Java Code Examples for org.apache.flume.Event#setHeaders()
The following examples show how to use
org.apache.flume.Event#setHeaders() .
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
| Source File: Consumer.java From rabbitmq-flume-plugin with BSD 3-Clause "New" or "Revised" License | 6 votes |
|
private Event parseMessage(Envelope envelope, AMQP.BasicProperties props, byte[] body) {
// Create the event passing in the body
Event event = EventBuilder.withBody(body);
// Get the headers from properties, exchange, and routing-key
Map<String, String> headers = buildHeaders(props);
String exchange = envelope.getExchange();
if (exchange != null && !exchange.isEmpty()) {
headers.put("exchange", exchange);
}
String routingKey = envelope.getRoutingKey();
if (routingKey != null && !routingKey.isEmpty()) {
headers.put("routing-key", routingKey);
}
event.setHeaders(headers);
return event;
}
Example 2
| Source File: Consumer.java From rabbitmq-flume-plugin with BSD 3-Clause "New" or "Revised" License | 6 votes |
|
private Event parseMessage(Envelope envelope, AMQP.BasicProperties props, byte[] body) {
// Create the event passing in the body
Event event = EventBuilder.withBody(body);
// Get the headers from properties, exchange, and routing-key
Map<String, String> headers = buildHeaders(props);
String exchange = envelope.getExchange();
if (exchange != null && !exchange.isEmpty()) {
headers.put("exchange", exchange);
}
String routingKey = envelope.getRoutingKey();
if (routingKey != null && !routingKey.isEmpty()) {
headers.put("routing-key", routingKey);
}
event.setHeaders(headers);
return event;
}
Example 3
| Source File: WatchDir.java From flume-taildirectory-source with Apache License 2.0 | 6 votes |
|
private void sendEvent(FileSet fileSet) {
LOGGER.trace("WatchDir: sendEvent");
if (fileSet.getBufferList().isEmpty())
return;
StringBuilder sb = fileSet.getAllLines();
Event event = EventBuilder.withBody(String.valueOf(sb).getBytes(),
fileSet.getHeaders());
Map<String,String> headers = new HashMap<String, String>();
if (fileHeader)
headers.put(fileHeaderKey,fileSet.getFilePath().toString());
if (basenameHeader)
headers.put(basenameHeaderKey, fileSet.getFileName().toString());
if (!headers.isEmpty())
event.setHeaders(headers);
source.getChannelProcessor().processEvent(event);
counter.increaseCounterMessageSent();
fileSet.clear();
}
Example 4
| Source File: GenerateSearchAnalyticsDataImpl.java From searchanalytics-bigdata with MIT License | 6 votes |
|
public Event getJsonEvent(
final SearchQueryInstruction searchQueryInstruction)
throws JsonProcessingException {
final String searchQueryInstructionAsString = getObjectMapper()
.writeValueAsString(searchQueryInstruction);
// String writeValueAsString =
// mapper.writerWithDefaultPrettyPrinter().writeValueAsString(searchQueryInstruction);
searchEventsLogger.info(searchQueryInstructionAsString);
final Event event = new JSONEvent();
event.setBody(searchQueryInstructionAsString.getBytes());
final Map<String, String> headers = new HashMap<String, String>();
headers.put("eventId", searchQueryInstruction.getEventIdSuffix());
headers.put("timestamp", searchQueryInstruction
.getCreatedTimeStampInMillis().toString());
if (searchQueryInstruction.getClickedDocId() != null) {
if (searchQueryInstruction.getFavourite() != null
&& searchQueryInstruction.getFavourite()) {
headers.put("State", "FAVOURITE");
} else {
headers.put("State", "VIEWED");
}
}
event.setHeaders(headers);
return event;
}
Example 5
| Source File: ElasticSearchJsonBodyEventSerializerTest.java From searchanalytics-bigdata with MIT License | 5 votes |
|
@Test
public void testESJsonEventSerializer() throws IOException {
final Event event = new JSONEvent();
final String writeValueAsString = "{\"hostedmachinename\":\"172.16.9.582\",\"pageurl\":\"http://blahblah:/1881\",\"customerid\":376,\"sessionid\":\"1eaa6cd1-0a71-4d03-aea4-d038921f5c6a\",\"querystring\":null,\"sortorder\":\"asc\",\"pagenumber\":0,\"totalhits\":39,\"hitsshown\":11,\"timestamp\":1397220014988,\"clickeddocid\":null,\"filters\":[{\"code\":\"specification_resolution\",\"value\":\"1024 x 600\"},{\"code\":\"searchfacettype_product_type_level_2\",\"value\":\"Laptops\"}]}";
event.setBody(writeValueAsString.getBytes());
final Map<String, String> headers = new HashMap<String, String>();
headers.put("eventId", UUID.randomUUID().toString());
event.setHeaders(headers);
((XContentBuilder) esSerializer.getContentBuilder(event)).string();
}
Example 6
| Source File: EventBuilder.java From mt-flume with Apache License 2.0 | 5 votes |
|
/**
* Instantiate an Event instance based on the provided body and headers.
* If <code>headers</code> is <code>null</code>, then it is ignored.
* @param body
* @param headers
* @return
*/
public static Event withBody(byte[] body, Map<String, String> headers) {
Event event = new SimpleEvent();
if(body == null) {
body = new byte[0];
}
event.setBody(body);
if (headers != null) {
event.setHeaders(new HashMap<String, String>(headers));
}
return event;
}
Example 7
| Source File: TestHeaderIndexBuilder.java From flume-elasticsearch-sink with Apache License 2.0 | 5 votes |
|
/**
* tests header based index, type and id
*/
@Test
public void testHeaderIndex() {
Event event = new SimpleEvent();
Map<String, String> headers = new HashMap<>();
headers.put(INDEX, index);
headers.put(TYPE, type);
headers.put(ID, id);
event.setHeaders(headers);
assertEquals(index, headerBasedIndexBuilder.getIndex(event));
assertEquals(type, headerBasedIndexBuilder.getType(event));
assertEquals(id, headerBasedIndexBuilder.getId(event));
}
Example 8
| Source File: ElasticSearchSerializerWithMappingTest.java From ingestion with Apache License 2.0 | 5 votes |
|
private Event createExampleEvent(long timestamp) {
String message = "test body";
Map<String, String> headers = Maps.newHashMap();
headers.put(TIMESTAMP_HEADER, String.valueOf(timestamp));
Event event = EventBuilder.withBody(message.getBytes(charset));
event.setHeaders(headers);
return event;
}
Example 9
| Source File: TestElasticSearchDynamicSerializer.java From ingestion with Apache License 2.0 | 5 votes |
|
@Test
public void testRoundTrip() throws Exception {
ElasticSearchDynamicSerializer fixture = new ElasticSearchDynamicSerializer();
Context context = new Context();
fixture.configure(context);
String message = "test body";
Map<String, String> headers = Maps.newHashMap();
headers.put("headerNameOne", "headerValueOne");
headers.put("headerNameTwo", "headerValueTwo");
headers.put("headerNameThree", "headerValueThree");
Event event = EventBuilder.withBody(message.getBytes(charset));
event.setHeaders(headers);
XContentBuilder expected = jsonBuilder().startObject();
expected.field("body", new String(message.getBytes(), charset));
for (String headerName : headers.keySet()) {
expected.field(headerName, new String(headers.get(headerName).getBytes(),
charset));
}
expected.endObject();
XContentBuilder actual = fixture.getContentBuilder(event);
assertEquals(new String(expected.bytes().array()), new String(actual
.bytes().array()));
}
Example 10
| Source File: TestMultiplexingChannelSelector.java From mt-flume with Apache License 2.0 | 5 votes |
|
@Test
public void testNoMandatory() {
config.put("default", "ch3");
config.put("optional.foo", "ch1 ch2");
config.put("optional.zebra", "ch2 ch3");
selector = ChannelSelectorFactory.create(channels, config);
Assert.assertTrue(selector instanceof MultiplexingChannelSelector);
Event event1 = new MockEvent();
Map<String, String> header1 = new HashMap<String, String>();
header1.put("myheader", "foo");// should match ch1 ch2
event1.setHeaders(header1);
List<Channel> reqCh1 = selector.getRequiredChannels(event1);
Assert.assertEquals(1, reqCh1.size());
Assert.assertEquals("ch3", reqCh1.get(0).getName());
List<Channel> optCh1 = selector.getOptionalChannels(event1);
Assert.assertEquals(2, optCh1.size());
//ch2 should not be there -- since it is a required channel
Assert.assertEquals("ch1", optCh1.get(0).getName());
Assert.assertEquals("ch2", optCh1.get(1).getName());
Event event4 = new MockEvent();
Map<String, String> header4 = new HashMap<String, String>();
header4.put("myheader", "zebra");
event4.setHeaders(header4);
List<Channel> reqCh4 = selector.getRequiredChannels(event4);
Assert.assertEquals(1, reqCh4.size());
Assert.assertTrue(reqCh4.get(0).getName().equals("ch3"));
List<Channel> optCh4 = selector.getOptionalChannels(event4);
//ch3 was returned as a required channel, because it is default.
//So it is not returned in optional
Assert.assertEquals(1, optCh4.size());
Assert.assertEquals("ch2", optCh4.get(0).getName());
}
Example 11
| Source File: TestFormatSpeed.java From mt-flume with Apache License 2.0 | 5 votes |
|
@Before
public void setUp(){
events = new ArrayList<Event>();
Event event = new SimpleEvent();
Map<String, String> headers = new HashMap<String, String>();
headers.put("category", "test");
event.setHeaders(headers);
event.setBody("".getBytes());
for(int i = 0; i < 200000; i++){
events.add(event);
}
}
Example 12
| Source File: TestElasticSearchDynamicSerializer.java From mt-flume with Apache License 2.0 | 5 votes |
|
@Test
public void testRoundTrip() throws Exception {
ElasticSearchDynamicSerializer fixture = new ElasticSearchDynamicSerializer();
Context context = new Context();
fixture.configure(context);
String message = "test body";
Map<String, String> headers = Maps.newHashMap();
headers.put("headerNameOne", "headerValueOne");
headers.put("headerNameTwo", "headerValueTwo");
headers.put("headerNameThree", "headerValueThree");
Event event = EventBuilder.withBody(message.getBytes(charset));
event.setHeaders(headers);
XContentBuilder expected = jsonBuilder().startObject();
expected.field("body", new String(message.getBytes(), charset));
for (String headerName : headers.keySet()) {
expected.field(headerName, new String(headers.get(headerName).getBytes(),
charset));
}
expected.endObject();
XContentBuilder actual = fixture.getContentBuilder(event);
assertEquals(new String(expected.bytes().array()), new String(actual
.bytes().array()));
}
Example 13
| Source File: TestElasticSearchLogStashEventSerializer.java From ingestion with Apache License 2.0 | 4 votes |
|
@Test
public void testRoundTrip() throws Exception {
ElasticSearchLogStashEventSerializer fixture = new ElasticSearchLogStashEventSerializer();
Context context = new Context();
fixture.configure(context);
String message = "test body";
Map<String, String> headers = Maps.newHashMap();
long timestamp = System.currentTimeMillis();
headers.put("timestamp", String.valueOf(timestamp));
headers.put("source", "flume_tail_src");
headers.put("host", "test@localhost");
headers.put("src_path", "/tmp/test");
headers.put("headerNameOne", "headerValueOne");
headers.put("headerNameTwo", "headerValueTwo");
headers.put("type", "sometype");
Event event = EventBuilder.withBody(message.getBytes(charset));
event.setHeaders(headers);
XContentBuilder expected = jsonBuilder()
.startObject();
expected.field("@message", new String(message.getBytes(), charset));
expected.field("@timestamp", new Date(timestamp));
expected.field("@source", "flume_tail_src");
expected.field("@type", "sometype");
expected.field("@source_host", "test@localhost");
expected.field("@source_path", "/tmp/test");
expected.startObject("@fields");
expected.field("timestamp", String.valueOf(timestamp));
expected.field("src_path", "/tmp/test");
expected.field("host", "test@localhost");
expected.field("headerNameTwo", "headerValueTwo");
expected.field("source", "flume_tail_src");
expected.field("headerNameOne", "headerValueOne");
expected.field("type", "sometype");
expected.endObject();
expected.endObject();
XContentBuilder actual = fixture.getContentBuilder(event);
JsonParser parser = new JsonParser();
assertEquals(parser.parse(expected.string()),parser.parse(actual.string()));
}
Example 14
| Source File: TestElasticSearchLogStashEventSerializer.java From mt-flume with Apache License 2.0 | 4 votes |
|
@Test
public void shouldHandleInvalidJSONDuringComplexParsing() throws Exception {
ElasticSearchLogStashEventSerializer fixture = new ElasticSearchLogStashEventSerializer();
Context context = new Context();
fixture.configure(context);
String message = "{flume: somethingnotvalid}";
Map<String, String> headers = Maps.newHashMap();
long timestamp = System.currentTimeMillis();
headers.put("timestamp", String.valueOf(timestamp));
headers.put("source", "flume_tail_src");
headers.put("host", "test@localhost");
headers.put("src_path", "/tmp/test");
headers.put("headerNameOne", "headerValueOne");
headers.put("headerNameTwo", "headerValueTwo");
headers.put("type", "sometype");
Event event = EventBuilder.withBody(message.getBytes(charset));
event.setHeaders(headers);
XContentBuilder expected = jsonBuilder().startObject();
expected.field("@message", new String(message.getBytes(), charset));
expected.field("@timestamp", new Date(timestamp));
expected.field("@source", "flume_tail_src");
expected.field("@type", "sometype");
expected.field("@source_host", "test@localhost");
expected.field("@source_path", "/tmp/test");
expected.startObject("@fields");
expected.field("timestamp", String.valueOf(timestamp));
expected.field("src_path", "/tmp/test");
expected.field("host", "test@localhost");
expected.field("headerNameTwo", "headerValueTwo");
expected.field("source", "flume_tail_src");
expected.field("headerNameOne", "headerValueOne");
expected.field("type", "sometype");
expected.endObject();
expected.endObject();
XContentBuilder actual = fixture.getContentBuilder(event);
assertEquals(new String(expected.bytes().array()), new String(actual
.bytes().array()));
}
Example 15
| Source File: TestElasticSearchLogStashEventSerializer.java From mt-flume with Apache License 2.0 | 4 votes |
|
@Test
public void testRoundTrip() throws Exception {
ElasticSearchLogStashEventSerializer fixture = new ElasticSearchLogStashEventSerializer();
Context context = new Context();
fixture.configure(context);
String message = "test body";
Map<String, String> headers = Maps.newHashMap();
long timestamp = System.currentTimeMillis();
headers.put("timestamp", String.valueOf(timestamp));
headers.put("source", "flume_tail_src");
headers.put("host", "test@localhost");
headers.put("src_path", "/tmp/test");
headers.put("headerNameOne", "headerValueOne");
headers.put("headerNameTwo", "headerValueTwo");
headers.put("type", "sometype");
Event event = EventBuilder.withBody(message.getBytes(charset));
event.setHeaders(headers);
XContentBuilder expected = jsonBuilder().startObject();
expected.field("@message", new String(message.getBytes(), charset));
expected.field("@timestamp", new Date(timestamp));
expected.field("@source", "flume_tail_src");
expected.field("@type", "sometype");
expected.field("@source_host", "test@localhost");
expected.field("@source_path", "/tmp/test");
expected.startObject("@fields");
expected.field("timestamp", String.valueOf(timestamp));
expected.field("src_path", "/tmp/test");
expected.field("host", "test@localhost");
expected.field("headerNameTwo", "headerValueTwo");
expected.field("source", "flume_tail_src");
expected.field("headerNameOne", "headerValueOne");
expected.field("type", "sometype");
expected.endObject();
expected.endObject();
XContentBuilder actual = fixture.getContentBuilder(event);
assertEquals(new String(expected.bytes().array()), new String(actual
.bytes().array()));
}
Example 16
| Source File: TestElasticSearchLogStashEventSerializer.java From ingestion with Apache License 2.0 | 4 votes |
|
@Test
public void shouldHandleInvalidJSONDuringComplexParsing() throws Exception {
ElasticSearchLogStashEventSerializer fixture = new ElasticSearchLogStashEventSerializer();
Context context = new Context();
fixture.configure(context);
String message = "{flume: somethingnotvalid}";
Map<String, String> headers = Maps.newHashMap();
long timestamp = System.currentTimeMillis();
headers.put("timestamp", String.valueOf(timestamp));
headers.put("source", "flume_tail_src");
headers.put("host", "test@localhost");
headers.put("src_path", "/tmp/test");
headers.put("headerNameOne", "headerValueOne");
headers.put("headerNameTwo", "headerValueTwo");
headers.put("type", "sometype");
Event event = EventBuilder.withBody(message.getBytes(charset));
event.setHeaders(headers);
XContentBuilder expected = jsonBuilder().
startObject();
expected.field("@message", new String(message.getBytes(), charset));
expected.field("@timestamp", new Date(timestamp));
expected.field("@source", "flume_tail_src");
expected.field("@type", "sometype");
expected.field("@source_host", "test@localhost");
expected.field("@source_path", "/tmp/test");
expected.startObject("@fields");
expected.field("timestamp", String.valueOf(timestamp));
expected.field("src_path", "/tmp/test");
expected.field("host", "test@localhost");
expected.field("headerNameTwo", "headerValueTwo");
expected.field("source", "flume_tail_src");
expected.field("headerNameOne", "headerValueOne");
expected.field("type", "sometype");
expected.endObject();
expected.endObject();
XContentBuilder actual = fixture.getContentBuilder(event);
JsonParser parser = new JsonParser();
assertEquals(parser.parse(expected.string()),parser.parse(actual.string()));
}
Example 17
| Source File: TestMultiplexingChannelSelector.java From mt-flume with Apache License 2.0 | 4 votes |
|
@Test
public void testNoDefault() {
config.put("mapping.foo", "ch1 ch2");
config.put("mapping.bar", "ch2 ch3");
config.put("mapping.xyz", "ch1 ch2 ch3");
config.put("mapping.zebra", "ch2");
config.put("optional.zebra", "ch1 ch3");
selector = ChannelSelectorFactory.create(channels, config);
Assert.assertTrue(selector instanceof MultiplexingChannelSelector);
Event event1 = new MockEvent();
Map<String, String> header1 = new HashMap<String, String>();
header1.put("myheader", "foo");// should match ch1 ch2
event1.setHeaders(header1);
List<Channel> reqCh1 = selector.getRequiredChannels(event1);
Assert.assertEquals(2, reqCh1.size());
Assert.assertEquals("ch1", reqCh1.get(0).getName());
Assert.assertEquals("ch2", reqCh1.get(1).getName());
List<Channel> optCh1 = selector.getOptionalChannels(event1);
Assert.assertTrue(optCh1.size() == 1);
//ch2 should not be there -- since it is a required channel
Assert.assertEquals("ch3", optCh1.get(0).getName());
Event event2 = new MockEvent();
Map<String, String> header2 = new HashMap<String, String>();
header2.put("myheader", "bar"); // should match ch2 ch3
event2.setHeaders(header2);
List<Channel> reqCh2 = selector.getRequiredChannels(event2);
Assert.assertEquals(2, reqCh2.size());
Assert.assertEquals("ch2", reqCh2.get(0).getName());
Assert.assertEquals("ch3", reqCh2.get(1).getName());
List<Channel> optCh2 = selector.getOptionalChannels(event2);
Assert.assertTrue(optCh2.isEmpty());
Event event3 = new MockEvent();
Map<String, String> header3 = new HashMap<String, String>();
header3.put("myheader", "xyz"); // should match ch1 ch2 ch3
event3.setHeaders(header3);
List<Channel> reqCh3 = selector.getRequiredChannels(event3);
Assert.assertEquals(3, reqCh3.size());
Assert.assertEquals("ch1", reqCh3.get(0).getName());
Assert.assertEquals("ch2", reqCh3.get(1).getName());
Assert.assertEquals("ch3", reqCh3.get(2).getName());
List<Channel> optCh3 = selector.getOptionalChannels(event3);
//All of the optional channels should go away.
Assert.assertTrue(optCh3.isEmpty());
Event event4 = new MockEvent();
Map<String, String> header4 = new HashMap<String, String>();
header4.put("myheader", "zebra");
event4.setHeaders(header4);
List<Channel> reqCh4 = selector.getRequiredChannels(event4);
Assert.assertEquals(1, reqCh4.size());
Assert.assertEquals("ch2", reqCh4.get(0).getName());
List<Channel> optCh4 = selector.getOptionalChannels(event4);
Assert.assertEquals(2, optCh4.size());
Assert.assertEquals("ch1", optCh4.get(0).getName());
Assert.assertEquals("ch3", optCh4.get(1).getName());
}
Example 18
| Source File: TestMultiplexingChannelSelector.java From mt-flume with Apache License 2.0 | 4 votes |
|
@Test
public void testNoSelection() throws Exception {
config.put("mapping.foo", "ch1 ch2");
config.put("mapping.bar", "ch2 ch3");
config.put("mapping.xyz", "ch1 ch2 ch3");
config.put("default", "ch1 ch3");
selector = ChannelSelectorFactory.create(channels, config);
Assert.assertTrue(selector instanceof MultiplexingChannelSelector);
Event noHeaderEvent = new MockEvent();
List<Channel> reqCh1 = selector.getRequiredChannels(noHeaderEvent);
List<Channel> optCh1 = selector.getOptionalChannels(noHeaderEvent);
Assert.assertEquals(2, reqCh1.size());
Assert.assertTrue(reqCh1.get(0).getName().equals("ch1"));
Assert.assertTrue(reqCh1.get(1).getName().equals("ch3"));
Assert.assertTrue(optCh1.isEmpty());
Map<String, String> header2 = new HashMap<String, String>();
header2.put("someheader", "foo");
Event invalidHeaderEvent = new MockEvent();
invalidHeaderEvent.setHeaders(header2);
List<Channel> reqCh2 = selector.getRequiredChannels(invalidHeaderEvent);
List<Channel> optCh2 = selector.getOptionalChannels(invalidHeaderEvent);
Assert.assertEquals(2, reqCh2.size());
Assert.assertTrue(reqCh2.get(0).getName().equals("ch1"));
Assert.assertTrue(reqCh2.get(1).getName().equals("ch3"));
Assert.assertTrue(optCh2.isEmpty());
Map<String, String> header3 = new HashMap<String, String>();
header3.put("myheader", "bar1");
Event unmatchedHeaderEvent = new MockEvent();
unmatchedHeaderEvent.setHeaders(header3);
List<Channel> reqCh3 = selector.getRequiredChannels(unmatchedHeaderEvent);
List<Channel> optCh3 = selector.getOptionalChannels(unmatchedHeaderEvent);
Assert.assertEquals(2, reqCh3.size());
Assert.assertTrue(reqCh3.get(0).getName().equals("ch1"));
Assert.assertTrue(reqCh3.get(1).getName().equals("ch3"));
Assert.assertTrue(optCh3.isEmpty());
Map<String, String> header4 = new HashMap<String, String>();
header4.put("myheader", "zebra");
Event zebraEvent = new MockEvent();
zebraEvent.setHeaders(header4);
List<Channel> reqCh4 = selector.getRequiredChannels(zebraEvent);
List<Channel> optCh4 = selector.getOptionalChannels(zebraEvent);
Assert.assertEquals(2, reqCh4.size());
Assert.assertTrue(reqCh4.get(0).getName().equals("ch1"));
Assert.assertTrue(reqCh4.get(1).getName().equals("ch3"));
//Since ch1 is also in default list, it is removed.
Assert.assertTrue(optCh4.size() == 1);
Assert.assertTrue(optCh4.get(0).getName().equals("ch2"));
List<Channel> allChannels = selector.getAllChannels();
Assert.assertTrue(allChannels.size() == 3);
Assert.assertTrue(allChannels.get(0).getName().equals("ch1"));
Assert.assertTrue(allChannels.get(1).getName().equals("ch2"));
Assert.assertTrue(allChannels.get(2).getName().equals("ch3"));
}
Example 19
| Source File: TestMultiplexingChannelSelector.java From mt-flume with Apache License 2.0 | 4 votes |
|
@Test
public void testSelection() throws Exception {
config.put("mapping.foo", "ch1 ch2");
config.put("mapping.bar", "ch2 ch3");
config.put("mapping.xyz", "ch1 ch2 ch3");
config.put("default", "ch1 ch3");
selector = ChannelSelectorFactory.create(channels, config);
Assert.assertTrue(selector instanceof MultiplexingChannelSelector);
Event event1 = new MockEvent();
Map<String, String> header1 = new HashMap<String, String>();
header1.put("myheader", "foo");// should match ch1 ch2
event1.setHeaders(header1);
List<Channel> reqCh1 = selector.getRequiredChannels(event1);
Assert.assertEquals(2, reqCh1.size());
Assert.assertTrue(reqCh1.get(0).getName().equals("ch1"));
Assert.assertTrue(reqCh1.get(1).getName().equals("ch2"));
List<Channel> optCh1 = selector.getOptionalChannels(event1);
Assert.assertTrue(optCh1.size() == 1);
//ch2 should not be there -- since it is a required channel
Assert.assertTrue(optCh1.get(0).getName().equals("ch3"));
Event event2 = new MockEvent();
Map<String, String> header2 = new HashMap<String, String>();
header2.put("myheader", "bar"); // should match ch2 ch3
event2.setHeaders(header2);
List<Channel> reqCh2 = selector.getRequiredChannels(event2);
Assert.assertEquals(2, reqCh2.size());
Assert.assertTrue(reqCh2.get(0).getName().equals("ch2"));
Assert.assertTrue(reqCh2.get(1).getName().equals("ch3"));
List<Channel> optCh2 = selector.getOptionalChannels(event2);
Assert.assertTrue(optCh2.isEmpty());
Event event3 = new MockEvent();
Map<String, String> header3 = new HashMap<String, String>();
header3.put("myheader", "xyz"); // should match ch1 ch2 ch3
event3.setHeaders(header3);
List<Channel> reqCh3 = selector.getRequiredChannels(event3);
Assert.assertEquals(3, reqCh3.size());
Assert.assertTrue(reqCh3.get(0).getName().equals("ch1"));
Assert.assertTrue(reqCh3.get(1).getName().equals("ch2"));
Assert.assertTrue(reqCh3.get(2).getName().equals("ch3"));
List<Channel> optCh3 = selector.getOptionalChannels(event3);
//All of the optional channels should go away.
Assert.assertTrue(optCh3.size() == 0);
}
Example 20
| Source File: AvroCLIClient.java From mt-flume with Apache License 2.0 | 4 votes |
|
private void run() throws IOException, FlumeException,
EventDeliveryException {
EventReader reader = null;
RpcClient rpcClient;
if (rpcClientPropsFile != null) {
rpcClient = RpcClientFactory.getInstance(new File(rpcClientPropsFile));
} else {
rpcClient = RpcClientFactory.getDefaultInstance(hostname, port,
BATCH_SIZE);
}
try {
if (fileName != null) {
reader = new SimpleTextLineEventReader(new FileReader(new File(fileName)));
} else if (dirName != null) {
reader = new ReliableSpoolingFileEventReader.Builder()
.spoolDirectory(new File(dirName)).build();
} else {
reader = new SimpleTextLineEventReader(new InputStreamReader(System.in));
}
long lastCheck = System.currentTimeMillis();
long sentBytes = 0;
int batchSize = rpcClient.getBatchSize();
List<Event> events;
while (!(events = reader.readEvents(batchSize)).isEmpty()) {
for (Event event : events) {
event.setHeaders(headers);
sentBytes += event.getBody().length;
sent++;
long now = System.currentTimeMillis();
if (now >= lastCheck + 5000) {
logger.debug("Packed {} bytes, {} events", sentBytes, sent);
lastCheck = now;
}
}
rpcClient.appendBatch(events);
if (reader instanceof ReliableEventReader) {
((ReliableEventReader) reader).commit();
}
}
logger.debug("Finished");
} finally {
if (reader != null) {
logger.debug("Closing reader");
reader.close();
}
logger.debug("Closing RPC client");
rpcClient.close();
}
}
