org.apache.flume.event.SimpleEvent Java Examples

@Test
public void shouldSetIndexNameTypeFromHeaderWhenPresent()
    throws Exception {
  String indexPrefix = "%{index-name}";
  String indexType = "%{index-type}";
  String indexValue = "testing-index-name-from-headers";
  String typeValue = "testing-index-type-from-headers";

  Event event = new SimpleEvent();
  event.getHeaders().put("index-name", indexValue);
  event.getHeaders().put("index-type", typeValue);

  IndexRequestBuilder indexRequestBuilder = factory.createIndexRequest(
      null, indexPrefix, indexType, event);

  assertEquals(indexValue + '-'
      + ElasticSearchIndexRequestBuilderFactory.df.format(FIXED_TIME_MILLIS),
    indexRequestBuilder.request().index());
  assertEquals(typeValue, indexRequestBuilder.request().type());
}
 

@Test
public void shouldSetIndexNameTypeAndSerializedEventIntoIndexRequest()
    throws Exception {

  String indexPrefix = "qwerty";
  String indexType = "uiop";
  Event event = new SimpleEvent();

  IndexRequestBuilder indexRequestBuilder = factory.createIndexRequest(
      FAKE_CLIENT, indexPrefix, indexType, event);

  assertEquals(indexPrefix + '-'
      + ElasticSearchIndexRequestBuilderFactory.df.format(FIXED_TIME_MILLIS),
    indexRequestBuilder.request().index());
  assertEquals(indexType, indexRequestBuilder.request().type());
  assertArrayEquals(FakeEventSerializer.FAKE_BYTES,
      indexRequestBuilder.request().source().array());
}
 

@Override
protected void append(ILoggingEvent logEvent) {
	SimpleEvent flumeEvent = new SimpleEvent();
	
	Map<String, String> headers = new HashMap<>();
	headers.put("timestamp", "" + logEvent.getTimeStamp());
	headers.put("level", logEvent.getLevel().levelStr);
	headers.put("threadName", logEvent.getThreadName());
	headers.put("loggerName", logEvent.getLoggerName());
	headers.put("product", product);
	headers.put("hostname", NetworkingUtils.getIpAddress());
	flumeEvent.setHeaders(headers);
	flumeEvent.setBody(logEvent.getFormattedMessage().getBytes(Charsets.UTF_8));
	
	try {
		agent.put(flumeEvent);
	} catch (Exception e) {
		e.printStackTrace();
	}
}
 

@Test
public void shouldSetIndexNameTypeAndSerializedEventIntoIndexRequest()
    throws Exception {

  String indexPrefix = "qwerty";
  String indexType = "uiop";
  Event event = new SimpleEvent();

  IndexRequestBuilder indexRequestBuilder = factory.createIndexRequest(
      FAKE_CLIENT, indexPrefix, indexType, event);

  assertEquals(indexPrefix + '-'
      + ElasticSearchIndexRequestBuilderFactory.df.format(FIXED_TIME_MILLIS),
    indexRequestBuilder.request().index());
  assertEquals(indexType, indexRequestBuilder.request().type());
  assertArrayEquals(FakeEventSerializer.FAKE_BYTES,
      indexRequestBuilder.request().source().array());
}
 

@Test
public void shouldUseExistingAtTimestampHeaderInTimestampedEvent() {
  SimpleEvent base = new SimpleEvent();
  Map<String, String> headersWithTimestamp = Maps.newHashMap();
  headersWithTimestamp.put("@timestamp", "-999");
  base.setHeaders(headersWithTimestamp );

  TimestampedEvent timestampedEvent = new TimestampedEvent(base);
  assertEquals(-999L, timestampedEvent.getTimestamp());
  assertEquals("-999", timestampedEvent.getHeaders().get("@timestamp"));
  assertNull(timestampedEvent.getHeaders().get("timestamp"));
}
 

@Test
public void testPrefix() throws Exception {
  Context context = new Context();
  context.put(UUIDInterceptor.HEADER_NAME, ID);
  context.put(UUIDInterceptor.PREFIX_NAME, "bar#");
  Event event = new SimpleEvent();
  assertTrue(build(context).intercept(event).getHeaders().get(ID).startsWith("bar#"));
}
 

@Test
public void shouldEnsureTimestampHeaderPresentInTimestampedEvent() {
  SimpleEvent base = new SimpleEvent();

  TimestampedEvent timestampedEvent = new TimestampedEvent(base);
  assertEquals(FIXED_TIME_MILLIS, timestampedEvent.getTimestamp());
  assertEquals(String.valueOf(FIXED_TIME_MILLIS),
      timestampedEvent.getHeaders().get("timestamp"));
}
 

@Test
public void shouldUseExistingTimestampHeaderInTimestampedEvent() {
  SimpleEvent base = new SimpleEvent();
  Map<String, String> headersWithTimestamp = Maps.newHashMap();
  headersWithTimestamp.put("timestamp", "-321");
  base.setHeaders(headersWithTimestamp );

  TimestampedEvent timestampedEvent = new TimestampedEvent(base);
  assertEquals(-321L, timestampedEvent.getTimestamp());
  assertEquals("-321", timestampedEvent.getHeaders().get("timestamp"));
}
 

@Test
public void shouldUseExistingAtTimestampHeaderInTimestampedEvent() {
  SimpleEvent base = new SimpleEvent();
  Map<String, String> headersWithTimestamp = Maps.newHashMap();
  headersWithTimestamp.put("@timestamp", "-999");
  base.setHeaders(headersWithTimestamp );

  TimestampedEvent timestampedEvent = new TimestampedEvent(base);
  assertEquals(-999L, timestampedEvent.getTimestamp());
  assertEquals("-999", timestampedEvent.getHeaders().get("@timestamp"));
  assertNull(timestampedEvent.getHeaders().get("timestamp"));
}
 

@Test
public void shouldPreserveBodyAndNonTimestampHeadersInTimestampedEvent() {
  SimpleEvent base = new SimpleEvent();
  base.setBody(new byte[] {1,2,3,4});
  Map<String, String> headersWithTimestamp = Maps.newHashMap();
  headersWithTimestamp.put("foo", "bar");
  base.setHeaders(headersWithTimestamp );

  TimestampedEvent timestampedEvent = new TimestampedEvent(base);
  assertEquals("bar", timestampedEvent.getHeaders().get("foo"));
  assertArrayEquals(base.getBody(), timestampedEvent.getBody());
}
 

@Test
public void shouldSetIndexNameFromTimestampHeaderWhenPresent()
    throws Exception {
  String indexPrefix = "qwerty";
  String indexType = "uiop";
  Event event = new SimpleEvent();
  event.getHeaders().put("timestamp", "1213141516");

  IndexRequestBuilder indexRequestBuilder = factory.createIndexRequest(
      null, indexPrefix, indexType, event);

  assertEquals(indexPrefix + '-'
      + ElasticSearchIndexRequestBuilderFactory.df.format(1213141516L),
    indexRequestBuilder.request().index());
}
 

@Test
public void testPut() throws EventDeliveryException {
  Event event = new SimpleEvent();
  agent.configure(properties);
  agent.start();
  agent.put(event);
  verify(source, times(1)).put(event);
}
 

@Test
public void testPutAll() throws EventDeliveryException {
  Event event = new SimpleEvent();
  List<Event> events = Lists.newArrayList();
  events.add(event);
  agent.configure(properties);
  agent.start();
  agent.putAll(events);
  verify(source, times(1)).putAll(events);
}
 

@Test(expected = IllegalStateException.class)
public void testPutAllNotStarted() throws EventDeliveryException {
  Event event = new SimpleEvent();
  List<Event> events = Lists.newArrayList();
  events.add(event);
  agent.configure(properties);
  agent.putAll(events);
}
 

@Test
public void shouldEnsureTimestampHeaderPresentInTimestampedEvent() {
  SimpleEvent base = new SimpleEvent();

  TimestampedEvent timestampedEvent = new TimestampedEvent(base);
  assertEquals(FIXED_TIME_MILLIS, timestampedEvent.getTimestamp());
  assertEquals(String.valueOf(FIXED_TIME_MILLIS),
          timestampedEvent.getHeaders().get("timestamp"));
}
 

@Test
public void shouldUseExistingTimestampHeaderInTimestampedEvent() {
  SimpleEvent base = new SimpleEvent();
  Map<String, String> headersWithTimestamp = Maps.newHashMap();
  headersWithTimestamp.put("timestamp", "-321");
  base.setHeaders(headersWithTimestamp );

  TimestampedEvent timestampedEvent = new TimestampedEvent(base);
  assertEquals(-321L, timestampedEvent.getTimestamp());
  assertEquals("-321", timestampedEvent.getHeaders().get("timestamp"));
}
 

@Test
public void testPreserveExisting() throws Exception {
  Context context = new Context();
  context.put(UUIDInterceptor.HEADER_NAME, ID);
  context.put(UUIDInterceptor.PRESERVE_EXISTING_NAME, "true");
  Event event = new SimpleEvent();
  event.getHeaders().put(ID, "foo");
  assertEquals("foo", build(context).intercept(event).getHeaders().get(ID));
}
 

@Test
public void shouldPreserveBodyAndNonTimestampHeadersInTimestampedEvent() {
  SimpleEvent base = new SimpleEvent();
  base.setBody(new byte[] {1,2,3,4});
  Map<String, String> headersWithTimestamp = Maps.newHashMap();
  headersWithTimestamp.put("foo", "bar");
  base.setHeaders(headersWithTimestamp );

  TimestampedEvent timestampedEvent = new TimestampedEvent(base);
  assertEquals("bar", timestampedEvent.getHeaders().get("foo"));
  assertArrayEquals(base.getBody(), timestampedEvent.getBody());
}
 

@Test
public void indexNameShouldBePrefixDashFormattedTimestamp() {
  long time = 987654321L;
  Event event = new SimpleEvent();
  Map<String, String> headers = new HashMap<String, String>();
  headers.put("timestamp", Long.toString(time));
  event.setHeaders(headers);
  assertEquals("prefix-" + indexNameBuilder.getFastDateFormat().format(time),
      indexNameBuilder.getIndexName(event));
}
 

@Test
public void shouldEnsureTimestampHeaderPresentInTimestampedEvent() {
  SimpleEvent base = new SimpleEvent();

  TimestampedEvent timestampedEvent = new TimestampedEvent(base);
  assertEquals(FIXED_TIME_MILLIS, timestampedEvent.getTimestamp());
  assertEquals(String.valueOf(FIXED_TIME_MILLIS),
      timestampedEvent.getHeaders().get("timestamp"));
}
 

@Test
public void shouldUseExistingTimestampHeaderInTimestampedEvent() {
  SimpleEvent base = new SimpleEvent();
  Map<String, String> headersWithTimestamp = Maps.newHashMap();
  headersWithTimestamp.put("timestamp", "-321");
  base.setHeaders(headersWithTimestamp );

  TimestampedEvent timestampedEvent = new TimestampedEvent(base);
  assertEquals(-321L, timestampedEvent.getTimestamp());
  assertEquals("-321", timestampedEvent.getHeaders().get("timestamp"));
}
 

@Test
public void shouldUseExistingAtTimestampHeaderInTimestampedEvent() {
  SimpleEvent base = new SimpleEvent();
  Map<String, String> headersWithTimestamp = Maps.newHashMap();
  headersWithTimestamp.put("@timestamp", "-999");
  base.setHeaders(headersWithTimestamp );

  TimestampedEvent timestampedEvent = new TimestampedEvent(base);
  assertEquals(-999L, timestampedEvent.getTimestamp());
  assertEquals("-999", timestampedEvent.getHeaders().get("@timestamp"));
  assertNull(timestampedEvent.getHeaders().get("timestamp"));
}
 

@Test
public void shouldPreserveBodyAndNonTimestampHeadersInTimestampedEvent() {
  SimpleEvent base = new SimpleEvent();
  base.setBody(new byte[] {1,2,3,4});
  Map<String, String> headersWithTimestamp = Maps.newHashMap();
  headersWithTimestamp.put("foo", "bar");
  base.setHeaders(headersWithTimestamp );

  TimestampedEvent timestampedEvent = new TimestampedEvent(base);
  assertEquals("bar", timestampedEvent.getHeaders().get("foo"));
  assertArrayEquals(base.getBody(), timestampedEvent.getBody());
}
 

@Test
public void shouldSetIndexNameFromTimestampHeaderWhenPresent()
    throws Exception {
  String indexPrefix = "qwerty";
  String indexType = "uiop";
  Event event = new SimpleEvent();
  event.getHeaders().put("timestamp", "1213141516");

  IndexRequestBuilder indexRequestBuilder = factory.createIndexRequest(
      null, indexPrefix, indexType, event);

  assertEquals(indexPrefix + '-'
      + ElasticSearchIndexRequestBuilderFactory.df.format(1213141516L),
    indexRequestBuilder.request().index());
}
 

private SimpleEvent createEvent(final DatabaseEntry data) {
    final SimpleEvent event = new SimpleEvent();
    try {
        byte[] eventData = data.getData();
        if (secretKey != null) {
            final Cipher cipher = Cipher.getInstance("AES");
            cipher.init(Cipher.DECRYPT_MODE, secretKey);
            eventData = cipher.doFinal(eventData);
        }
        final ByteArrayInputStream bais = new ByteArrayInputStream(eventData);
        final DataInputStream dais = new DataInputStream(bais);
        int length = dais.readInt();
        final byte[] bytes = new byte[length];
        dais.read(bytes, 0, length);
        event.setBody(bytes);
        length = dais.readInt();
        final Map<String, String> map = new HashMap<>(length);
        for (int i = 0; i < length; ++i) {
            final String headerKey = dais.readUTF();
            final String value = dais.readUTF();
            map.put(headerKey, value);
        }
        event.setHeaders(map);
        return event;
    } catch (final Exception ex) {
        LOGGER.error("Error retrieving event", ex);
        return null;
    }
}
 

@Test
public void testBasic() throws Exception {
  Context context = new Context();
  context.put(UUIDInterceptor.HEADER_NAME, ID);
  context.put(UUIDInterceptor.PRESERVE_EXISTING_NAME, "true");
  Event event = new SimpleEvent();
  assertTrue(build(context).intercept(event).getHeaders().get(ID).length() > 0);
}
 

/**
 * tests configuration based index and type
 */
@Test
public void testConfigurationIndex() {
    Event event = new SimpleEvent();
    Context context = new Context();
    context.put(ES_INDEX, index);
    context.put(ES_TYPE, type);
    headerBasedIndexBuilder.configure(context);
    assertEquals(index, headerBasedIndexBuilder.getIndex(event));
    assertEquals(type, headerBasedIndexBuilder.getType(event));
}
 

/**
 * tests Default index and type
 */
@Test
public void testDefaultIndex() {
    Event event = new SimpleEvent();
    assertEquals(DEFAULT_ES_INDEX, headerBasedIndexBuilder.getIndex(event));
    assertEquals(DEFAULT_ES_TYPE, headerBasedIndexBuilder.getType(event));
}
 

public static void main(String[] args) throws FileNotFoundException,
		IOException, EventDeliveryException {
	if (args.length == 0) {
		System.out
				.println("AvroClient {host} {port} {numOfEvents}");
		return;
	}

	String host = args[0];
	int port = Integer.parseInt(args[1]);
	int numberOfEvents = Integer.parseInt(args[2]);

	Properties starterProp = new Properties();
	starterProp.setProperty(RpcClientConfigurationConstants.CONFIG_HOSTS, "h1");
    starterProp.setProperty(RpcClientConfigurationConstants.CONFIG_HOSTS_PREFIX + "h1",  host + ":" + port);
    
	NettyAvroRpcClient client = (NettyAvroRpcClient) RpcClientFactory
			.getInstance(starterProp);

	System.out.println("Starting");
	for (int i = 0; i < numberOfEvents; i++) {

		if (i%100 == 0) {
			System.out.print(".");
		}
		
		SimpleEvent event = generateEvent(i);

		client.append(event);
	}
	System.out.println();
	System.out.println("Done");
}
 

protected static SimpleEvent generateEvent(int i) {
	SimpleEvent event = new SimpleEvent();

	char c1 = (char) (i % 26 + 65);
	char c2 = (char) (System.currentTimeMillis() % 26 + 65);

	String body = "Event body " + c1 + " " + c2;
	event.setBody(body.getBytes());
	return event;
}