Java Code Examples for org.apache.flume.event.EventBuilder#withBody()

@Test
public void testSizeRoller() throws IOException, InterruptedException {
  int maxBytes = 300;
  MockHDFSWriter hdfsWriter = new MockHDFSWriter();
  BucketWriter bucketWriter = new BucketWriter(0, maxBytes, 0, 0, ctx,
      "/tmp", "file", "", ".tmp", null, null, SequenceFile.CompressionType.NONE,
      hdfsWriter, timedRollerPool, null,
      new SinkCounter("test-bucket-writer-" + System.currentTimeMillis()),
      0, null, null, 30000, Executors.newSingleThreadExecutor());

  Event e = EventBuilder.withBody("foo", Charsets.UTF_8);
  for (int i = 0; i < 1000; i++) {
    bucketWriter.append(e);
  }

  logger.info("Number of events written: {}", hdfsWriter.getEventsWritten());
  logger.info("Number of bytes written: {}", hdfsWriter.getBytesWritten());
  logger.info("Number of files opened: {}", hdfsWriter.getFilesOpened());

  Assert.assertEquals("events written", 1000, hdfsWriter.getEventsWritten());
  Assert.assertEquals("bytes written", 3000, hdfsWriter.getBytesWritten());
  Assert.assertEquals("files opened", 10, hdfsWriter.getFilesOpened());
}
 

private Event getEvent(long offset) {
    ObjectNode jsonBody = new ObjectNode(JsonNodeFactory.instance);
    jsonBody.put("field1", "foo");
    jsonBody.put("field2", 32);
    jsonBody.put("timestamp", String.valueOf(new Date().getTime()));

    Map<String, String> headers = new HashMap<String, String>();
    headers.put("field3", "bar"); // Overwrites the value defined in JSON body
    headers.put("field4", "64");
    headers.put("field5", "true");
    headers.put("field6", "1.0");
    headers.put("field7", "11");
    final long l = new Date().getTime();
    headers.put("timestamp", String.valueOf(l + offset));

    headers.put("myString2", "baz");

    return EventBuilder.withBody(jsonBody.toString().getBytes(Charsets.UTF_8), headers);
}
 

private static List<Event> generateSyslogEvents() {
  List<Event> list = Lists.newArrayList();

  Event e;

  // generate one that we supposedly parsed with SyslogTcpSource
  e = EventBuilder.withBody("Apr  7 01:00:00 host Msg 01", Charsets.UTF_8);
  e.getHeaders().put(SyslogUtils.SYSLOG_FACILITY, "1");
  e.getHeaders().put(SyslogUtils.SYSLOG_SEVERITY, "2");
  list.add(e);

  // generate another supposedly parsed with SyslogTcpSource with 2-digit date
  e = EventBuilder.withBody("Apr 22 01:00:00 host Msg 02", Charsets.UTF_8);
  e.getHeaders().put(SyslogUtils.SYSLOG_FACILITY, "1");
  e.getHeaders().put(SyslogUtils.SYSLOG_SEVERITY, "3");
  list.add(e);

  // generate a "raw" syslog event
  e = EventBuilder.withBody("<8>Apr 22 01:00:00 host Msg 03", Charsets.UTF_8);
  list.add(e);

  return list;
}
 

/**
 * Ensure that the "host" header gets set (to something)
 */
@Test
public void testBasic() throws Exception {
  Interceptor.Builder builder = InterceptorBuilderFactory.newInstance(
          InterceptorType.HOST.toString());
  Interceptor interceptor = builder.build();

  Event eventBeforeIntercept = EventBuilder.withBody("test event",
          Charsets.UTF_8);
  Assert.assertNull(eventBeforeIntercept.getHeaders().get(Constants.HOST));

  Event eventAfterIntercept = interceptor.intercept(eventBeforeIntercept);
  String actualHost = eventAfterIntercept.getHeaders().get(Constants.HOST);

  Assert.assertNotNull(actualHost);
}
 

@Test
public void testInUsePrefix() throws IOException, InterruptedException {
  final int ROLL_INTERVAL = 1000; // seconds. Make sure it doesn't change in course of test
  final String PREFIX = "BRNO_IS_CITY_IN_CZECH_REPUBLIC";

  MockHDFSWriter hdfsWriter = new MockHDFSWriter();
  HDFSTextSerializer formatter = new HDFSTextSerializer();
  BucketWriter bucketWriter = new BucketWriter(ROLL_INTERVAL, 0, 0, 0, ctx,
      "/tmp", "file", PREFIX, ".tmp", null, null, SequenceFile.CompressionType.NONE, hdfsWriter,
      timedRollerPool, null,
      new SinkCounter("test-bucket-writer-" + System.currentTimeMillis()),
      0, null, null, 30000, Executors.newSingleThreadExecutor());

  Event e = EventBuilder.withBody("foo", Charsets.UTF_8);
  bucketWriter.append(e);

  Assert.assertTrue("Incorrect in use prefix", hdfsWriter.getOpenedFilePath().contains(PREFIX));
}
 

private Event getTrackerEvent() {
    Random random = new Random();
    String[] users = new String[] { "[email protected]", "[email protected]", "[email protected]",
            "[email protected]" };
    String[] isoCode = new String[] { "DE", "ES", "US", "FR" };
    TimeUnit[] offset = new TimeUnit[] { TimeUnit.DAYS, TimeUnit.HOURS, TimeUnit.SECONDS };
    ObjectNode jsonBody = new ObjectNode(JsonNodeFactory.instance);
    Map<String, String> headers;
    ObjectMapper mapper = new ObjectMapper();
    JsonNode jsonNode = null;
    final String fileName = "/trackerSample" + random.nextInt(4) + ".json";
    try {
        jsonNode = mapper.readTree(getClass().getResourceAsStream(fileName));
    } catch (IOException e) {
        e.printStackTrace();
    }
    headers = mapper.convertValue(jsonNode, Map.class);
    headers.put("timestamp", String.valueOf(new Date().getTime() + getOffset(offset[random.nextInt(3)]) * random
            .nextInt(100)));
    headers.put("santanderID", users[random.nextInt(4)]);
    headers.put("isoCode", isoCode[random.nextInt(4)]);

    return EventBuilder.withBody(jsonBody.toString().getBytes(Charsets.UTF_8), headers);
}
 

@Test
public void shouldIndexOneEvent() throws Exception {
  Configurables.configure(fixture, new Context(parameters));
  Channel channel = bindAndStartChannel(fixture);

  Transaction tx = channel.getTransaction();
  tx.begin();
  Event event = EventBuilder.withBody("event #1 or 1".getBytes());
  channel.put(event);
  tx.commit();
  tx.close();

  fixture.process();
  fixture.stop();
  client.admin().indices()
      .refresh(Requests.refreshRequest(timestampedIndexName)).actionGet();

  assertMatchAllQuery(1, event);
  assertBodyQuery(1, event);
}
 

@Override
public Void append( AvroFlumeOGEvent evt ) throws AvroRemoteException {
  counterGroup.incrementAndGet("rpc.received");
  Map<String, String> headers = new HashMap<String, String>();

  // extract Flume OG event headers
  headers.put(HOST, evt.getHost().toString());
  headers.put(TIMESTAMP, evt.getTimestamp().toString());
  headers.put(PRIORITY, evt.getPriority().toString());
  headers.put(NANOS, evt.getNanos().toString());
  for (Entry<CharSequence, ByteBuffer> entry : evt.getFields().entrySet()) {
    headers.put(entry.getKey().toString(), entry.getValue().toString());
  }
  headers.put(OG_EVENT, "yes");

  Event event = EventBuilder.withBody(evt.getBody().array(), headers);
  try {
    getChannelProcessor().processEvent(event);
    counterGroup.incrementAndGet("rpc.events");
  } catch (ChannelException ex) {
    return null;
  }

  counterGroup.incrementAndGet("rpc.successful");
  return null;
}
 

@Test
public void testCommit() throws InterruptedException, EventDeliveryException {

  Event event, event2;
  Context context = new Context();
  int putCounter = 0;

  context.put("keep-alive", "1");
  context.put("capacity", "100");
  context.put("transactionCapacity", "50");
  Configurables.configure(channel, context);

  Transaction transaction = channel.getTransaction();
  Assert.assertNotNull(transaction);

  transaction.begin();
  for (putCounter = 0; putCounter < 10; putCounter++) {
    event = EventBuilder.withBody(("test event" + putCounter).getBytes());
    channel.put(event);
  }
  transaction.commit();
  transaction.close();

  transaction = channel.getTransaction();
  Assert.assertNotNull(transaction);

  transaction = channel.getTransaction();
  transaction.begin();
  for (int i = 0; i < 10; i++) {
    event2 = channel.take();
    Assert.assertNotNull("lost an event", event2);
    Assert.assertArrayEquals(event2.getBody(), ("test event" + i).getBytes());
    // System.out.println(event2.toString());
  }
  event2 = channel.take();
  Assert.assertNull("extra event found", event2);

  transaction.commit();
  transaction.close();
}
 

public Event poll() {
  AvroFlumeEvent avroEvent = eventQueue.poll();
  if(avroEvent != null) {
    return EventBuilder.withBody(avroEvent.getBody().array(),
        toStringMap(avroEvent.getHeaders()));
  }
  return null;
}
 

@Test(expected = FlumeException.class)
public void testMissingTable() throws Exception {
  deleteTable = false;
  ctx.put("batchSize", "2");
  AsyncHBaseSink sink = new AsyncHBaseSink(testUtility.getConfiguration());
  Configurables.configure(sink, ctx);
  //Reset the context to a higher batchSize
  ctx.put("batchSize", "100");
  Channel channel = new MemoryChannel();
  Configurables.configure(channel, ctx);
  sink.setChannel(channel);
  sink.start();
  Transaction tx = channel.getTransaction();
  tx.begin();
  for(int i = 0; i < 3; i++){
    Event e = EventBuilder.withBody(Bytes.toBytes(valBase + "-" + i));
    channel.put(e);
  }
  tx.commit();
  tx.close();
  sink.process();
  Assert.assertFalse(sink.isConfNull());
  HTable table = new HTable(testUtility.getConfiguration(), tableName);
  byte[][] results = getResults(table, 2);
  byte[] out;
  int found = 0;
  for(int i = 0; i < 2; i++){
    for(int j = 0; j < 2; j++){
      if(Arrays.equals(results[j],Bytes.toBytes(valBase + "-" + i))){
        found++;
        break;
      }
    }
  }
  Assert.assertEquals(2, found);
  out = results[2];
  Assert.assertArrayEquals(Longs.toByteArray(2), out);
  sink.process();
  sink.stop();
}
 

public void append(ThriftFlumeEvent evt ) {
  if (evt == null) {
    return;
  }

  Map<String, String> headers = new HashMap<String, String>();
  // extract Flume event headers
  headers.put(HOST, evt.getHost());
  headers.put(TIMESTAMP, Long.toString(evt.getTimestamp()));
  headers.put(PRIORITY, evt.getPriority().toString());
  headers.put(NANOS, Long.toString(evt.getNanos()));
  for (Entry<String, ByteBuffer> entry: evt.getFields().entrySet()) {
    headers.put(entry.getKey().toString(),
      UTF_8.decode(entry.getValue()).toString());
  }
  headers.put(OG_EVENT, "yes");

  Event event = EventBuilder.withBody(evt.getBody(), headers);
  counterGroup.incrementAndGet("rpc.events");
  try {
    getChannelProcessor().processEvent(event);
  } catch (ChannelException ex) {
    LOG.warn("Failed to process event", ex);
    return;
  }

  counterGroup.incrementAndGet("rpc.successful");
  return;
}
 

@Test
public void testRoundTrip() throws Exception {
  ElasticSearchDynamicSerializer fixture = new ElasticSearchDynamicSerializer();
  Context context = new Context();
  fixture.configure(context);

  String message = "test body";
  Map<String, String> headers = Maps.newHashMap();
  headers.put("headerNameOne", "headerValueOne");
  headers.put("headerNameTwo", "headerValueTwo");
  headers.put("headerNameThree", "headerValueThree");
  Event event = EventBuilder.withBody(message.getBytes(charset));
  event.setHeaders(headers);

  XContentBuilder expected = jsonBuilder().startObject();
  expected.field("body", new String(message.getBytes(), charset));
  for (String headerName : headers.keySet()) {
    expected.field(headerName, new String(headers.get(headerName).getBytes(),
        charset));
  }
  expected.endObject();

  XContentBuilder actual = fixture.getContentBuilder(event);

  assertEquals(new String(expected.bytes().array()), new String(actual
      .bytes().array()));

}
 

public void sendData(String data){
    Event event = EventBuilder.withBody(data.getBytes());

    try {
        rpcClient.append(event);
    } catch (EventDeliveryException e) {
        e.printStackTrace();
        rpcClient.close();
        rpcClient = RpcClientFactory.getDefaultInstance(this.hostName, this.port);
    }

}
 

@Test
public void noFullBatch() throws Exception {
    setField(mongoSink, "batchSize", 5);
    for (int i = 0; i < 3; i++) {
        Transaction tx = channel.getTransaction();
        tx.begin();
        Event event = EventBuilder.withBody("{ }".getBytes(Charsets.UTF_8));
        channel.put(event);
        tx.commit();
        tx.close();
    }
    mongoSink.process();
    assertThat(fongo.getDB("test").getCollection("test").count()).isEqualTo(3);
}
 

@Test
public void shouldIndexFiveEventsOverThreeBatches() throws Exception {
  parameters.put(BATCH_SIZE, "2");
  Configurables.configure(fixture, new Context(parameters));
  Channel channel = bindAndStartChannel(fixture);

  int numberOfEvents = 5;
  Event[] events = new Event[numberOfEvents];

  Transaction tx = channel.getTransaction();
  tx.begin();
  for (int i = 0; i < numberOfEvents; i++) {
    String body = "event #" + i + " of " + numberOfEvents;
    Event event = EventBuilder.withBody(body.getBytes());
    events[i] = event;
    channel.put(event);
  }
  tx.commit();
  tx.close();

  int count = 0;
  Status status = Status.READY;
  while (status != Status.BACKOFF) {
    count++;
    status = fixture.process();
  }
  fixture.stop();

  assertEquals(3, count);

  client.admin().indices()
      .refresh(Requests.refreshRequest(timestampedIndexName)).actionGet();
  assertMatchAllQuery(numberOfEvents, events);
  assertBodyQuery(5, events);
}
 

@Test
/* leading XXXXX does not match regex, thus we expect the event to be dropped */
public void testGrokIfNotMatchDropEventDrop() throws Exception {
  Context context = new Context();
  context.put(MorphlineHandlerImpl.MORPHLINE_FILE_PARAM, RESOURCES_DIR + "/test-morphlines/grokIfNotMatchDropRecord.conf");
  String msg = "<XXXXXXXXXXXXX164>Feb  4 10:46:14 syslog sshd[607]: Server listening on 0.0.0.0 port 22.";
  Event input = EventBuilder.withBody(null, ImmutableMap.of(Fields.MESSAGE, msg));
  Event actual = build(context).intercept(input);
  assertNull(actual);
}
 

@Test
public void testFailedConnect() throws Exception {

  Event event = EventBuilder.withBody("test event 1",
    Charset.forName("UTF8"));

  sink.start();

  Thread.sleep(500L); // let socket startup
  Thread.sleep(500L); // sleep a little to allow close occur

  Transaction transaction = channel.getTransaction();
  transaction.begin();
  for (int i = 0; i < 10; i++) {
    channel.put(event);
  }
  transaction.commit();
  transaction.close();

  for (int i = 0; i < 5; i++) {
    boolean threwException = false;
    try {
      sink.process();
    } catch (EventDeliveryException e) {
      threwException = true;
    }
    Assert.assertTrue("Must throw EventDeliveryException if disconnected",
      threwException);
  }

  src = new ThriftTestingSource(ThriftTestingSource.HandlerType.OK.name(),
    port);

  for (int i = 0; i < 5; i++) {
    Sink.Status status = sink.process();
    Assert.assertEquals(Sink.Status.READY, status);
  }

  Assert.assertEquals(Sink.Status.BACKOFF, sink.process());
  sink.stop();
}
 

private Event buildEvent(String json) {
    Event event = EventBuilder.withBody(json, Charsets.UTF_8, new HashMap<String, String>());
    return event;
}
 

public void createAvroFile(File file, String codec, boolean useSchemaUrl) throws
    IOException {

  // serialize a few events using the reflection-based avro serializer
  OutputStream out = new FileOutputStream(file);

  Context ctx = new Context();
  if (codec != null) {
    ctx.put("compressionCodec", codec);
  }

  Schema schema = Schema.createRecord("myrecord", null, null, false);
  schema.setFields(Arrays.asList(new Schema.Field[]{
      new Schema.Field("message", Schema.create(Schema.Type.STRING), null, null)
  }));
  GenericRecordBuilder recordBuilder = new GenericRecordBuilder(schema);
  File schemaFile = null;
  if (useSchemaUrl) {
    schemaFile = File.createTempFile(getClass().getSimpleName(), ".avsc");
    Files.write(schema.toString(), schemaFile, Charsets.UTF_8);
  }

  EventSerializer.Builder builder = new AvroEventSerializer.Builder();
  EventSerializer serializer = builder.build(ctx, out);

  serializer.afterCreate();
  for (int i = 0; i < 3; i++) {
    GenericRecord record = recordBuilder.set("message", "Hello " + i).build();
    Event event = EventBuilder.withBody(serializeAvro(record, schema));
    if (schemaFile == null) {
      event.getHeaders().put(AvroEventSerializer.AVRO_SCHEMA_LITERAL_HEADER,
          schema.toString());
    } else {
      event.getHeaders().put(AvroEventSerializer.AVRO_SCHEMA_URL_HEADER,
          schemaFile.toURI().toURL().toExternalForm());
    }
    serializer.write(event);
  }
  serializer.flush();
  serializer.beforeClose();
  out.flush();
  out.close();
}