Java Code Examples for nl.basjes.parse.core.Parser#parse()
The following examples show how to use
nl.basjes.parse.core.Parser#parse() .
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
| Source File: Main.java From logparser with Apache License 2.0 | 5 votes |
|
private void run() throws InvalidDissectorException, MissingDissectorsException, NoSuchMethodException, DissectionFailure {
// This format and logline originate from here:
// https://stackoverflow.com/questions/20349184/java-parse-log-file
String logformat = "%t %u [%D %h %{True-Client-IP}i %{UNIQUE_ID}e %r] %{Cookie}i %s \"%{User-Agent}i\" \"%{host}i\" %l %b %{Referer}i";
String logline = "[02/Dec/2013:14:10:30 -0000] - [52075 10.102.4.254 177.43.52.210 UpyU1gpmBAwAACfd5W0AAAAW GET /SS14-VTam-ny_019.j" +
"pg.rendition.zoomable.jpg HTTP/1.1] hsfirstvisit=http%3A%2F%2Fwww.domain.com%2Fen-us||1372268254000; _opt_vi_3FNG8DZU=F870" +
"DCFD-CBA4-4B6E-BB58-4605A78EE71A; __ptca=145721067.0aDxsZlIuM48.1372279055.1379945057.1379950362.9; __ptv_62vY4e=0aDxsZlIu" +
"M48; __pti_62vY4e=0aDxsZlIuM48; __ptcz=145721067.1372279055.1.0.ptmcsr=(direct)|ptmcmd=(none)|ptmccn=(direct); __hstc=1457" +
"21067.b86362bb7a1d257bfa2d1fb77e128a85.1372268254968.1379934256743.1379939561848.9; hubspotutk=b86362bb7a1d257bfa2d1fb77e1" +
"28a85; USER_GROUP=julinho%3Afalse; has_js=1; WT_FPC=id=177.43.52.210-1491335248.30301337:lv=1385997780893:ss=1385997780893" +
"; dtCookie=1F2E0E1037589799D8D503EB8CFA12A1|_default|1; RM=julinho%3A5248423ad3fe062f06c54915e6cde5cb45147977; wcid=UpyKsQ" +
"pmBAwAABURyNoAAAAS%3A35d8227ba1e8a9a9cebaaf8d019a74777c32b4c8; Carte::KerberosLexicon_getWGSN=82ae3dcd1b956288c3c86bdbed6e" +
"bcc0fd040e1e; UserData=Username%3AJULINHO%3AHomepage%3A1%3AReReg%3A0%3ATrialist%3A0%3ALanguage%3Aen%3ACcode%3Abr%3AForceRe" +
"Reg%3A0; UserID=1356673%3A12345%3A1234567890%3A123%3Accode%3Abr; USER_DATA=1356673%3Ajulinho%3AJulio+Jose%3Ada+Silva%3Ajul" +
"inho%40tecnoblu.com.br%3A0%3A1%3Aen%3Abr%3A%3AWGSN%3A1385990833.81925%3A82ae3dcd1b956288c3c86bdbed6ebcc0fd040e1e; MODE=FON" +
"TIS; SECTION=%2Fcontent%2Fsection%2Fhome.html; edge_auth=ip%3D177.43.52.210~expires%3D1385994522~access%3D%2Fapps%2F%2A%21" +
"%2Fbin%2F%2A%21%2Fcontent%2F%2A%21%2Fetc%2F%2A%21%2Fhome%2F%2A%21%2Flibs%2F%2A%21%2Freport%2F%2A%21%2Fsection%2F%2A%21%2Fw" +
"gsn%2F%2A~md5%3D90e73ee10161c1afacab12c6ea30b4ef; __utma=94539802.1793276213.1372268248.1385572390.1385990581.16; __utmb=9" +
"4539802.52.9.1385991739764; __utmc=94539802; __utmz=94539802.1372268248.1.1.utmcsr=(direct)|utmccn=(direct)|utmcmd=(none);" +
" WT_FPC=id=177.43.52.210-1491335248.30301337:lv=1386000374581:ss=1386000374581; dtPC=-; NSC_wtfswfs_xfcgbsn40-41=ffffffff0" +
"96e1a1d45525d5f4f58455e445a4a423660; akamai-edge=5ac6e5b3d0bbe2ea771bb2916d8bab34ea222a6a 200 \"Mozilla/5.0 (Windows NT 6." +
"2; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/31.0.1650.57 Safari/537.36\" \"www.domain.com\" - 463952 http://ww" +
"w.domain.com/content/report/shows/New_York/KSHK/trip/s_s_14_ny_ww/sheers.html";
printAllPossibles(logformat);
Parser<MyRecord> parser = new HttpdLoglineParser<>(MyRecord.class, logformat);
MyRecord record = new MyRecord();
LOG.info("==================================================================================");
parser.parse(record, logline);
LOG.info(record.toString());
LOG.info("==================================================================================");
}
Example 2
| Source File: ApacheHttpdLogParserTest.java From logparser with Apache License 2.0 | 5 votes |
|
@Test
public void fullTest2() throws Exception {
Parser<TestRecord> parser = new HttpdLoglineParser<>(TestRecord.class, LOG_FORMAT);
String line = "%127.0.0.1 127.0.0.1 127.0.0.1 - - [10/Aug/2012:23:55:11 +0200] \"GET /icons/powered_by_rh.png HTTP/1.1\" 200 1213 80"
+ " \"\" \"http://localhost/\" 1306 \"Mozilla/5.0 (X11; Linux i686 on x86_64; rv:11.0) Gecko/20100101 Firefox/11.0\""
+ " \"jquery-ui-theme=Eggplant; Apache=127.0.0.1.1344635667182858\" \"-\" \"-\" \"\\\"3780ff-4bd-4c1ce3df91380\\\"\"";
TestRecord record = new TestRecord();
parser.parse(record, line);
Map<String, String> results = record.getResults();
assertEquals(null, results.get("HTTP.QUERYSTRING:request.firstline.uri.query.foo"));
assertEquals("127.0.0.1", results.get("IP:connection.client.ip"));
assertEquals(null, results.get("NUMBER:connection.client.logname"));
assertEquals(null, results.get("STRING:connection.client.user"));
assertEquals("10/Aug/2012:23:55:11 +0200", results.get("TIME.STAMP:request.receive.time"));
assertEquals("11", results.get("TIME.SECOND:request.receive.time.second"));
assertEquals("/icons/powered_by_rh.png", results.get("HTTP.URI:request.firstline.uri"));
assertEquals("200", results.get("STRING:request.status.last"));
assertEquals("1213", results.get("BYTESCLF:response.body.bytes"));
assertEquals("http://localhost/", results.get("HTTP.URI:request.referer"));
assertEquals("Mozilla/5.0 (X11; Linux i686 on x86_64; rv:11.0) Gecko/20100101 Firefox/11.0",
results.get("HTTP.USERAGENT:request.user-agent"));
assertEquals("10", results.get("TIME.DAY:request.receive.time.day"));
assertEquals("23", results.get("TIME.HOUR:request.receive.time.hour"));
assertEquals("August", results.get("TIME.MONTHNAME:request.receive.time.monthname"));
assertEquals("1306", results.get("MICROSECONDS:response.server.processing.time"));
assertEquals(null, results.get("HTTP.SETCOOKIES:response.cookies"));
assertEquals("jquery-ui-theme=Eggplant; Apache=127.0.0.1.1344635667182858",
results.get("HTTP.COOKIES:request.cookies"));
assertEquals("\\\"3780ff-4bd-4c1ce3df91380\\\"", results.get("HTTP.HEADER:response.header.etag"));
// assertEquals("351",results.get("COOKIE:request.cookie.jquery-ui-theme"));
}
Example 3
| Source File: ApacheHttpdLogParserTest.java From logparser with Apache License 2.0 | 5 votes |
|
@Test
public void fullTestTooLongUri() throws Exception {
Parser<TestRecord> parser = new HttpdLoglineParser<>(TestRecord.class, LOG_FORMAT);
String line = "%127.0.0.1 127.0.0.1 127.0.0.1 - - [10/Aug/2012:23:55:11 +0200] \"GET /ImagineAURLHereThatIsTooLong\" 414 1213 80"
+ " \"\" \"http://localhost/\" 1306 \"Mozilla/5.0 (X11; Linux i686 on x86_64; rv:11.0) Gecko/20100101 Firefox/11.0\""
+ " \"jquery-ui-theme=Eggplant; Apache=127.0.0.1.1344635667182858\" \"-\" \"-\" \"\\\"3780ff-4bd-4c1ce3df91380\\\"\"";
TestRecord record = new TestRecord();
parser.parse(record, line);
Map<String, String> results = record.getResults();
// System.out.println(results.toString());
assertEquals(null, results.get("HTTP.QUERYSTRING:request.firstline.uri.query.foo"));
assertEquals("127.0.0.1", results.get("IP:connection.client.ip"));
assertEquals(null, results.get("NUMBER:connection.client.logname"));
assertEquals(null, results.get("STRING:connection.client.user"));
assertEquals("10/Aug/2012:23:55:11 +0200", results.get("TIME.STAMP:request.receive.time"));
assertEquals("11", results.get("TIME.SECOND:request.receive.time.second"));
assertEquals("/ImagineAURLHereThatIsTooLong", results.get("HTTP.URI:request.firstline.uri"));
assertEquals("414", results.get("STRING:request.status.last"));
assertEquals("1213", results.get("BYTESCLF:response.body.bytes"));
assertEquals("http://localhost/", results.get("HTTP.URI:request.referer"));
assertEquals("Mozilla/5.0 (X11; Linux i686 on x86_64; rv:11.0) Gecko/20100101 Firefox/11.0",
results.get("HTTP.USERAGENT:request.user-agent"));
assertEquals("10", results.get("TIME.DAY:request.receive.time.day"));
assertEquals("23", results.get("TIME.HOUR:request.receive.time.hour"));
assertEquals("August", results.get("TIME.MONTHNAME:request.receive.time.monthname"));
assertEquals("1306", results.get("MICROSECONDS:response.server.processing.time"));
assertEquals(null, results.get("HTTP.SETCOOKIES:response.cookies"));
assertEquals("jquery-ui-theme=Eggplant; Apache=127.0.0.1.1344635667182858",
results.get("HTTP.COOKIES:request.cookies"));
assertEquals("\\\"3780ff-4bd-4c1ce3df91380\\\"", results.get("HTTP.HEADER:response.header.etag"));
// assertEquals("351",results.get("COOKIE:request.cookie.jquery-ui-theme"));
}
Example 4
| Source File: ApacheHttpdLogParserTest.java From logparser with Apache License 2.0 | 5 votes |
|
@Test
public void testMissing() throws Exception {
try {
Parser<TestRecordMissing> parser = new HttpdLoglineParser<>(TestRecordMissing.class, LOG_FORMAT);
parser.parse(""); // Just to trigger the internal assembly of things (that should fail).
fail("Missing exception.");
} catch (MissingDissectorsException e) {
assertTrue(e.getMessage().contains("HEADER:response.header.etag.thisshouldbemissing"));
}
}
Example 5
| Source File: ApacheHttpdLogParserTest.java From logparser with Apache License 2.0 | 5 votes |
|
@Test
public void testMissing2() throws Exception {
try {
Parser<TestRecordMissing2> parser = new HttpdLoglineParser<>(TestRecordMissing2.class, LOG_FORMAT);
parser.parse(""); // Just to trigger the internal assembly of things (that should fail).
fail("Missing exception.");
} catch (MissingDissectorsException e) {
assertTrue(e.getMessage().contains("BLURP:request.firstline.uri.query.thisshouldbemissing"));
}
}
Example 6
| Source File: ApacheHttpdLogParserTest.java From logparser with Apache License 2.0 | 5 votes |
|
/**
* Test of mod_reqtimeout 408 status code
* Assume mod_reqtimeout is enabled and absolutely no data is entered by a client
* after making the connection. The result is a http 408 status code and a logline that has proven to
* result in several fields failing to be parsed because they are different than the specifications.
*/
@Test
public void test408ModReqTimeout() throws Exception {
final String logformat =
"\"%%\" \"%a\" \"%{c}a\" \"%A\" \"%B\" \"%b\" \"%D\" \"%f\" \"%h\" \"%H\" \"%k\" " +
"\"%l\" \"%L\" \"%m\" \"%p\" \"%{canonical}p\" \"%{local}p\" \"%{remote}p\" \"%P\" \"%{pid}P\" \"%{tid}P\"" +
" \"%{hextid}P\" \"%q\" \"%r\" \"%R\" \"%s\" \"%>s\" \"%t\" \"%{msec}t\" \"%{begin:msec}t\" \"%{end:msec}t" +
"\" \"%{usec}t\" \"%{begin:usec}t\" \"%{end:usec}t\" \"%{msec_frac}t\" \"%{begin:msec_frac}t\" \"%{end:mse" +
"c_frac}t\" \"%{usec_frac}t\" \"%{begin:usec_frac}t\" \"%{end:usec_frac}t\" \"%T\" \"%u\" \"%U\" \"%v\" \"" +
"%V\" \"%X\" \"%I\" \"%O\" \"%{cookie}i\" \"%{set-cookie}o\" \"%{user-agent}i\" \"%{referer}i\"";
String line200 = "\"%\" \"127.0.0.1\" \"127.0.0.1\" \"127.0.0.1\" \"3186\" \"3186\" \"1302\" \"/var/www/html/index.html\" " +
"\"127.0.0.1\" \"HTTP/1.1\" \"0\" \"-\" \"-\" \"GET\" \"80\" \"80\" \"80\" \"50142\" \"10344\" \"10344\" " +
"\"139854162249472\" \"139854162249472\" \"\" \"GET / HTTP/1.1\" \"-\" \"200\" \"200\" " +
"\"[09/Aug/2016:22:57:59 +0200]\" \"1470776279833\" \"1470776279833\" \"1470776279835\" \"1470776279833934\" " +
"\"1470776279833934\" \"1470776279835236\" \"833\" \"833\" \"835\" \"833934\" \"833934\" \"835236\" \"0\" " +
"\"-\" \"/index.html\" \"committer.lan.basjes.nl\" \"localhost\" \"+\" \"490\" \"3525\" \"-\" \"-\" " +
"\"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/52.0.2743.116 Safari/537.36\" \"-\"";
String line408 = "\"%\" \"127.0.0.1\" \"127.0.0.1\" \"127.0.0.1\" \"0\" \"-\" \"34\" \"-\" " +
"\"127.0.0.1\" \"HTTP/1.0\" \"0\" \"-\" \"-\" \"-\" \"80\" \"80\" \"80\" \"50150\" \"10344\" \"10344\" " +
"\"139854067267328\" \"139854067267328\" \"\" \"-\" \"-\" \"408\" \"408\" " +
"\"[09/Aug/2016:22:59:14 +0200]\" \"1470776354625\" \"1470776354625\" \"1470776354625\" \"1470776354625377\" " +
"\"1470776354625377\" \"1470776354625411\" \"625\" \"625\" \"625\" \"625377\" \"625377\" \"625411\" \"0\" " +
"\"-\" \"-\" \"committer.lan.basjes.nl\" \"committer.lan.basjes.nl\" \"-\" \"0\" \"0\" \"-\" \"-\" \"-\" \"-\"";
Parser<EmptyTestRecord> parser =
new HttpdLoglineParser<>(EmptyTestRecord.class, logformat)
.addParseTarget(EmptyTestRecord.class.getMethod("put", String.class, String.class),
"STRING:request.firstline.uri.query.foo");
parser.parse(new EmptyTestRecord(), line200);
parser.parse(new EmptyTestRecord(), line408);
}
Example 7
| Source File: MultiLineHttpdLogParserTest.java From logparser with Apache License 2.0 | 5 votes |
|
private void validateLine1(Parser<TestRecord> parser) throws InvalidDissectorException, MissingDissectorsException, DissectionFailure {
TestRecord record = new TestRecord();
parser.parse(record, LINE_1);
Map<String, String> results = record.getResults();
assertEquals("127.0.0.1", results.get("IP:connection.client.host"));
assertEquals("31/Dec/2012:23:49:41 +0100", results.get("TIME.STAMP:request.receive.time"));
assertEquals("/foo", results.get("HTTP.URI:request.firstline.uri"));
assertEquals("200", results.get("STRING:request.status.last"));
assertEquals("1213", results.get("BYTESCLF:response.body.bytes"));
assertEquals("http://localhost/index.php?mies=wim", results.get("HTTP.URI:request.referer"));
assertEquals(null, results.get("HTTP.USERAGENT:request.user-agent"));
}
Example 8
| Source File: MultiLineHttpdLogParserTest.java From logparser with Apache License 2.0 | 5 votes |
|
private void validateLine2(Parser<TestRecord> parser) throws InvalidDissectorException, MissingDissectorsException, DissectionFailure {
TestRecord record = new TestRecord();
parser.parse(record, LINE_2);
Map<String, String> results = record.getResults();
assertEquals("127.0.0.2", results.get("IP:connection.client.host"));
assertEquals("31/Dec/2012:23:49:42 +0100", results.get("TIME.STAMP:request.receive.time"));
assertEquals("/foo", results.get("HTTP.URI:request.firstline.uri"));
assertEquals("404", results.get("STRING:request.status.last"));
assertEquals(null, results.get("BYTESCLF:response.body.bytes"));
assertEquals(null, results.get("HTTP.URI:request.referer"));
assertEquals("Mozilla/5.0 (X11; Linux i686 on x86_64; rv:11.0) Gecko/20100101 Firefox/11.0",
results.get("HTTP.USERAGENT:request.user-agent"));
}
Example 9
| Source File: BasicOverallTest.java From logparser with Apache License 2.0 | 5 votes |
|
@Test
public void testBasicParsing() throws Exception {
Parser<MyRecord> parser = new HttpdLoglineParser<>(MyRecord.class, LOG_FORMAT);
MyRecord record = new MyRecord();
List<String> paths = parser.getPossiblePaths();
parser.addParseTarget(record.getClass().getMethod("setValue", String.class, String.class), paths);
for (String logline : LOG_LINES) {
record.clear();
parser.parse(record, logline);
System.out.println(record.toString());
}
}
Example 10
| Source File: ApacheHttpdLogParserTest.java From logparser with Apache License 2.0 | 4 votes |
|
/**
* Test of initialize method, of class ApacheHttpdLogParser.
*/
@Test
public void fullTest1() throws Exception {
String line = "%127.0.0.1 127.0.0.1 127.0.0.1 - - [31/Dec/2012:23:49:40 +0100] "
+ "\"GET /icons/powered_by_rh.png?aap=noot&res=1024x768 HTTP/1.1\" 200 1213 "
+ "80 \"\" \"http://localhost/index.php?mies=wim\" 351 "
+ "\"Mozilla/5.0 (X11; Linux i686 on x86_64; rv:11.0) Gecko/20100101 Firefox/11.0\" "
+ "\"jquery-ui-theme=Eggplant\" \"Apache=127.0.0.1.1344635380111339; path=/; domain=.basjes.nl\" \"-\" "
+ "\"\\\"3780ff-4bd-4c1ce3df91380\\\"\"";
Parser<TestRecord> parser = new HttpdLoglineParser<>(TestRecord.class, LOG_FORMAT);
// Manually add an extra dissector
parser.addDissector(new ScreenResolutionDissector());
parser.addTypeRemapping("request.firstline.uri.query.res", "SCREENRESOLUTION");
List<String> extraFields = new ArrayList<>();
extraFields.add("SCREENWIDTH:request.firstline.uri.query.res.width");
extraFields.add("SCREENHEIGHT:request.firstline.uri.query.res.height");
parser.addParseTarget(TestRecord.class.getMethod("setValue", String.class, String.class), extraFields);
TestRecord record = new TestRecord();
parser.parse(record, line);
Map<String, String> results = record.getResults();
System.out.println(results.toString());
assertEquals("noot", results.get("STRING:request.firstline.uri.query.aap"));
assertEquals(null, results.get("STRING:request.firstline.uri.query.foo"));
assertEquals(null, results.get("STRING:request.querystring.aap"));
assertEquals("1024", results.get("SCREENWIDTH:request.firstline.uri.query.res.width"));
assertEquals("768", results.get("SCREENHEIGHT:request.firstline.uri.query.res.height"));
assertEquals("127.0.0.1", results.get("IP:connection.client.ip"));
assertEquals(null, results.get("NUMBER:connection.client.logname"));
assertEquals(null, results.get("STRING:connection.client.user"));
assertEquals("31/Dec/2012:23:49:40 +0100", results.get("TIME.STAMP:request.receive.time"));
assertEquals("1356994180000", results.get("TIME.EPOCH:request.receive.time.epoch"));
assertEquals("1", results.get("TIME.WEEK:request.receive.time.weekofweekyear"));
assertEquals("2013", results.get("TIME.YEAR:request.receive.time.weekyear"));
assertEquals("2012", results.get("TIME.YEAR:request.receive.time.year"));
assertEquals("40", results.get("TIME.SECOND:request.receive.time.second"));
assertEquals("/icons/powered_by_rh.png?aap=noot&res=1024x768", results.get("HTTP.URI:request.firstline.uri"));
assertEquals("200", results.get("STRING:request.status.last"));
assertEquals("1213", results.get("BYTESCLF:response.body.bytes"));
assertEquals("http://localhost/index.php?mies=wim", results.get("HTTP.URI:request.referer"));
assertEquals("wim", results.get("STRING:request.referer.query.mies"));
assertEquals("Mozilla/5.0 (X11; Linux i686 on x86_64; rv:11.0) Gecko/20100101 Firefox/11.0",
results.get("HTTP.USERAGENT:request.user-agent"));
assertEquals("31", results.get("TIME.DAY:request.receive.time.day"));
assertEquals("23", results.get("TIME.HOUR:request.receive.time.hour"));
assertEquals("December", results.get("TIME.MONTHNAME:request.receive.time.monthname"));
assertEquals("351", results.get("MICROSECONDS:response.server.processing.time"));
assertEquals("Apache=127.0.0.1.1344635380111339; path=/; domain=.basjes.nl",
results.get("HTTP.SETCOOKIES:response.cookies"));
assertEquals("jquery-ui-theme=Eggplant", results.get("HTTP.COOKIES:request.cookies"));
assertEquals("\\\"3780ff-4bd-4c1ce3df91380\\\"", results.get("HTTP.HEADER:response.header.etag"));
assertEquals("Eggplant", results.get("HTTP.COOKIE:request.cookies.jquery-ui-theme"));
assertEquals("Apache=127.0.0.1.1344635380111339; path=/; domain=.basjes.nl", results.get("HTTP.SETCOOKIE:response.cookies.apache"));
assertEquals(".basjes.nl", results.get("STRING:response.cookies.apache.domain"));
}
Example 11
| Source File: ApacheHttpdLogParserTest.java From logparser with Apache License 2.0 | 4 votes |
|
@Test
public void testQueryStringDissector() throws Exception {
String logformat = "%r";
Parser<EmptyTestRecord> parser = new HttpdLoglineParser<>(EmptyTestRecord.class, logformat);
String[] params = {"STRING:request.firstline.uri.query.foo",
"STRING:request.firstline.uri.query.bar",
"HTTP.PATH:request.firstline.uri.path",
"HTTP.QUERYSTRING:request.firstline.uri.query",
"HTTP.REF:request.firstline.uri.ref"
};
parser.addParseTarget(EmptyTestRecord.class.getMethod("put", String.class, String.class), Arrays.asList(params));
EmptyTestRecord record = new EmptyTestRecord();
parser.parse(record, "GET /index.html HTTP/1.1");
assertEquals(null, record.get("STRING:request.firstline.uri.query.foo"));
assertEquals(null, record.get("STRING:request.firstline.uri.query.bar"));
assertEquals("/index.html", record.get("HTTP.PATH:request.firstline.uri.path"));
assertEquals("", record.get("HTTP.QUERYSTRING:request.firstline.uri.query"));
assertEquals(null, record.get("HTTP.REF:request.firstline.uri.ref"));
record.clear();
parser.parse(record, "GET /index.html?foo HTTP/1.1");
assertEquals("", record.get("STRING:request.firstline.uri.query.foo"));
assertEquals(null, record.get("STRING:request.firstline.uri.query.bar"));
assertEquals("/index.html", record.get("HTTP.PATH:request.firstline.uri.path"));
assertEquals("&foo", record.get("HTTP.QUERYSTRING:request.firstline.uri.query"));
assertEquals(null, record.get("HTTP.REF:request.firstline.uri.ref"));
record.clear();
parser.parse(record, "GET /index.html&foo HTTP/1.1");
assertEquals("", record.get("STRING:request.firstline.uri.query.foo"));
assertEquals(null, record.get("STRING:request.firstline.uri.query.bar"));
assertEquals("/index.html", record.get("HTTP.PATH:request.firstline.uri.path"));
assertEquals("&foo", record.get("HTTP.QUERYSTRING:request.firstline.uri.query"));
assertEquals(null, record.get("HTTP.REF:request.firstline.uri.ref"));
record.clear();
parser.parse(record, "GET /index.html?foo=foofoo# HTTP/1.1");
assertEquals("foofoo", record.get("STRING:request.firstline.uri.query.foo"));
assertEquals(null, record.get("STRING:request.firstline.uri.query.bar"));
assertEquals("/index.html", record.get("HTTP.PATH:request.firstline.uri.path"));
assertEquals("&foo=foofoo", record.get("HTTP.QUERYSTRING:request.firstline.uri.query"));
assertEquals("", record.get("HTTP.REF:request.firstline.uri.ref"));
record.clear();
parser.parse(record, "GET /index.html&foo=foofoo HTTP/1.1");
assertEquals("foofoo", record.get("STRING:request.firstline.uri.query.foo"));
assertEquals(null, record.get("STRING:request.firstline.uri.query.bar"));
assertEquals("/index.html", record.get("HTTP.PATH:request.firstline.uri.path"));
assertEquals("&foo=foofoo", record.get("HTTP.QUERYSTRING:request.firstline.uri.query"));
assertEquals(null, record.get("HTTP.REF:request.firstline.uri.ref"));
record.clear();
parser.parse(record, "GET /index.html?bar&foo=foofoo# HTTP/1.1");
assertEquals("foofoo", record.get("STRING:request.firstline.uri.query.foo"));
assertEquals("", record.get("STRING:request.firstline.uri.query.bar"));
assertEquals("/index.html", record.get("HTTP.PATH:request.firstline.uri.path"));
assertEquals("&bar&foo=foofoo", record.get("HTTP.QUERYSTRING:request.firstline.uri.query"));
assertEquals("", record.get("HTTP.REF:request.firstline.uri.ref"));
record.clear();
parser.parse(record, "GET /index.html?bar&foo=foofoo#bookmark HTTP/1.1");
assertEquals("foofoo", record.get("STRING:request.firstline.uri.query.foo"));
assertEquals("", record.get("STRING:request.firstline.uri.query.bar"));
assertEquals("/index.html", record.get("HTTP.PATH:request.firstline.uri.path"));
assertEquals("&bar&foo=foofoo", record.get("HTTP.QUERYSTRING:request.firstline.uri.query"));
assertEquals("bookmark", record.get("HTTP.REF:request.firstline.uri.ref"));
record.clear();
parser.parse(record, "GET /index.html?bar=barbar&foo=foofoo#bookmark HTTP/1.1");
assertEquals("foofoo", record.get("STRING:request.firstline.uri.query.foo"));
assertEquals("barbar", record.get("STRING:request.firstline.uri.query.bar"));
assertEquals("/index.html", record.get("HTTP.PATH:request.firstline.uri.path"));
assertEquals("&bar=barbar&foo=foofoo", record.get("HTTP.QUERYSTRING:request.firstline.uri.query"));
assertEquals("bookmark", record.get("HTTP.REF:request.firstline.uri.ref"));
record.clear();
parser.parse(record, "GET /index.html&bar=barbar&foo=foofoo#bla HTTP/1.1");
assertEquals("foofoo", record.get("STRING:request.firstline.uri.query.foo"));
assertEquals("barbar", record.get("STRING:request.firstline.uri.query.bar"));
assertEquals("/index.html", record.get("HTTP.PATH:request.firstline.uri.path"));
assertEquals("&bar=barbar&foo=foofoo", record.get("HTTP.QUERYSTRING:request.firstline.uri.query"));
assertEquals("bla", record.get("HTTP.REF:request.firstline.uri.ref"));
record.clear();
parser.parse(record, "GET /index.html&bar=barbar?foo=foofoo HTTP/1.1");
assertEquals("foofoo", record.get("STRING:request.firstline.uri.query.foo"));
assertEquals("barbar", record.get("STRING:request.firstline.uri.query.bar"));
assertEquals("/index.html", record.get("HTTP.PATH:request.firstline.uri.path"));
assertEquals("&bar=barbar&foo=foofoo", record.get("HTTP.QUERYSTRING:request.firstline.uri.query"));
assertEquals(null, record.get("HTTP.REF:request.firstline.uri.ref"));
}
Example 12
| Source File: CookiesTest.java From logparser with Apache License 2.0 | 4 votes |
|
@Test
public void cookiesTest() throws Exception {
Parser<TestRecord> parser = new HttpdLoglineParser<>(TestRecord.class, LOG_FORMAT);
TestRecord record = new TestRecord();
parser.parse(record, COOKIES_LINE);
// ---------------
Map<String, String> results = record.getResults();
Map<String, Long> longResults = record.getLongResults();
// System.out.println(results.toString());
assertEquals(null, results.get("QUERYSTRING:request.firstline.uri.query.foo"));
assertEquals("127.0.0.1", results.get("IP:connection.client.ip"));
assertEquals(null, results.get("NUMBER:connection.client.logname"));
assertEquals(null, results.get("STRING:connection.client.user"));
assertEquals("31/Dec/2012:23:00:44 -0700", results.get("TIME.STAMP:request.receive.time"));
assertEquals("1357020044000", results.get("TIME.EPOCH:request.receive.time.epoch"));
assertEquals(Long.valueOf(1357020044000L), longResults.get("TIME.EPOCH:request.receive.time.epoch"));
assertEquals("2012", results.get("TIME.YEAR:request.receive.time.year"));
assertEquals("12", results.get("TIME.MONTH:request.receive.time.month"));
assertEquals("December", results.get("TIME.MONTHNAME:request.receive.time.monthname"));
assertEquals("31", results.get("TIME.DAY:request.receive.time.day"));
assertEquals(Long.valueOf(31), longResults.get("TIME.DAY:request.receive.time.day"));
assertEquals("23", results.get("TIME.HOUR:request.receive.time.hour"));
assertEquals(Long.valueOf(23), longResults.get("TIME.HOUR:request.receive.time.hour"));
assertEquals("44", results.get("TIME.SECOND:request.receive.time.second"));
assertEquals(Long.valueOf(44), longResults.get("TIME.SECOND:request.receive.time.second"));
assertEquals("2013", results.get("TIME.YEAR:request.receive.time.year_utc"));
assertEquals("1", results.get("TIME.MONTH:request.receive.time.month_utc"));
assertEquals("January", results.get("TIME.MONTHNAME:request.receive.time.monthname_utc"));
assertEquals("1", results.get("TIME.DAY:request.receive.time.day_utc"));
assertEquals(Long.valueOf(1), longResults.get("TIME.DAY:request.receive.time.day_utc"));
assertEquals("6", results.get("TIME.HOUR:request.receive.time.hour_utc"));
assertEquals(Long.valueOf(6), longResults.get("TIME.HOUR:request.receive.time.hour_utc"));
assertEquals("44", results.get("TIME.SECOND:request.receive.time.second_utc"));
assertEquals(Long.valueOf(44), longResults.get("TIME.SECOND:request.receive.time.second_utc"));
assertEquals("/index.php", results.get("HTTP.URI:request.firstline.uri"));
assertEquals("200", results.get("STRING:request.status.last"));
// The "-" value means "Not specified" which is mapped to the setter being called
// with a 'null' value intending to say "We know it is not there".
assertTrue(results.containsKey("BYTESCLF:response.body.bytes"));
assertEquals(null, results.get("BYTESCLF:response.body.bytes"));
assertTrue(longResults.containsKey("BYTESCLF:response.body.bytes"));
assertEquals(null, longResults.get("BYTESCLF:response.body.bytes"));
assertEquals(null, results.get("HTTP.URI:request.referer"));
assertEquals("Mozilla/5.0 (X11; Linux i686 on x86_64; rv:11.0) Gecko/20100101 Firefox/11.0",
results.get("HTTP.USERAGENT:request.user-agent"));
assertEquals("80991", results.get("MICROSECONDS:response.server.processing.time"));
assertEquals(null, results.get("HTTP.HEADER:response.header.etag"));
assertEquals("Eggplant", results.get("HTTP.COOKIE:request.cookies.jquery-ui-theme"));
assertEquals("127.0.0.1.1351111543699529", results.get("HTTP.COOKIE:request.cookies.apache"));
assertEquals("NBA-0=, " +
"NBA-1=1234, " +
"NBA-2=1234; expires=Wed, 01-Jan-2020 00:00:10 GMT, " +
"NBA-3=1234; expires=Wed, 01-Jan-2020 00:00:10 GMT; path=/, " +
"NBA-4=1234; expires=Wed, 01-Jan-2020 00:00:10 GMT; path=/; domain=.basj.es",
results.get("HTTP.SETCOOKIES:response.cookies"));
assertEquals("NBA-4=1234; expires=Wed, 01-Jan-2020 00:00:10 GMT; path=/; domain=.basj.es",
results.get("HTTP.SETCOOKIE:response.cookies.nba-4"));
assertEquals("1234", results.get("STRING:response.cookies.nba-4.value"));
// The returned value may be off by 1 or 2 seconds due to rounding.
assertEquals(1577836810D, Double.parseDouble(results.get("STRING:response.cookies.nba-4.expires")), 2D);
assertEquals("/", results.get("STRING:response.cookies.nba-4.path"));
assertEquals(".basj.es", results.get("STRING:response.cookies.nba-4.domain"));
}
Example 13
| Source File: JettyLogFormatParserTest.java From logparser with Apache License 2.0 | 4 votes |
|
@Test
public void buggyJettyLogline() throws Exception {
// In Jetty
// - an extra space is included if the useragent is absent (the >"-" < near the end).
// - two extra spaces are included if the user field is absent ( " - " instead of "-" )
String[] lines = {
"0.0.0.0 - x [24/Jul/2016:07:08:31 +0000] \"GET http://[:1]/foo HTTP/1.1\" 400 0 \"http://other.site\" \"-\" 8",
"0.0.0.0 - - [24/Jul/2016:07:08:31 +0000] \"GET http://[:1]/foo HTTP/1.1\" 400 0 \"http://other.site\" \"-\" 8",
"0.0.0.0 - x [24/Jul/2016:07:08:31 +0000] \"GET http://[:1]/foo HTTP/1.1\" 400 0 \"http://other.site\" \"Mozilla/5.0 (dummy)\" 8",
"0.0.0.0 - - [24/Jul/2016:07:08:31 +0000] \"GET http://[:1]/foo HTTP/1.1\" 400 0 \"http://other.site\" \"Mozilla/5.0 (dummy)\" 8",
};
Parser<TestRecord> parser = new HttpdLoglineParser<>(TestRecord.class,
"ENABLE JETTY FIX\n"+
"%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\" %D"
);
for (String line:lines) {
TestRecord record = new TestRecord();
parser.parse(record, line);
Map<String, String> results = record.getResults();
assertEquals("0.0.0.0", results.get("IP:connection.client.host"));
assertEquals(null, results.get("NUMBER:connection.client.logname"));
String user = results.get("STRING:connection.client.user");
if (user!=null) {
assertEquals("x", user);
}
assertEquals("24/Jul/2016:07:08:31 +0000", results.get("TIME.STAMP:request.receive.time"));
assertEquals("24", results.get("TIME.DAY:request.receive.time.day"));
assertEquals("GET http://[:1]/foo HTTP/1.1", results.get("HTTP.FIRSTLINE:request.firstline"));
assertEquals("400", results.get("STRING:request.status.last"));
assertEquals("0", results.get("BYTES:response.body.bytes"));
assertEquals("http://other.site", results.get("HTTP.URI:request.referer"));
String useragent = results.get("HTTP.USERAGENT:request.user-agent");
if (useragent!=null) {
assertEquals("Mozilla/5.0 (dummy)", useragent);
}
assertEquals("8", results.get("MICROSECONDS:response.server.processing.time"));
}
}
Example 14
| Source File: TestFieldSetters.java From logparser with Apache License 2.0 | 4 votes |
|
@Test
public void testEmptyValues() throws InvalidDissectorException, MissingDissectorsException, DissectionFailure {
Parser<TestFieldSettersRecord> parser = new Parser<>(TestFieldSettersRecord.class);
parser.setRootType("INPUT");
parser.addDissector(new EmptyValuesDissector());
TestFieldSettersRecord testRecord = parser.parse("Doesn't matter");
testRecord
// Default (== Always)
.expectString( "D-ANY:any", "")
.expectString( "D-STRING:string", "")
.expectString( "D-INT:int", "")
.expectString( "D-LONG:long", "")
.expectString( "D-FLOAT:float", "")
.expectString( "D-DOUBLE:double", "")
.expectLong( "D-ANY:any", null)
.noLong( "D-STRING:string")
.expectLong( "D-INT:int", null)
.expectLong( "D-LONG:long", null)
.noLong( "D-FLOAT:float")
.noLong( "D-DOUBLE:double")
.expectDouble( "D-ANY:any", null)
.noDouble( "D-STRING:string")
.noDouble( "D-INT:int")
.noDouble( "D-LONG:long")
.expectDouble( "D-FLOAT:float", null)
.expectDouble( "D-DOUBLE:double", null)
// Always
.expectString( "A-ANY:any", "")
.expectString( "A-STRING:string", "")
.expectString( "A-INT:int", "")
.expectString( "A-LONG:long", "")
.expectString( "A-FLOAT:float", "")
.expectString( "A-DOUBLE:double", "")
.expectLong( "A-ANY:any", null)
.noLong( "A-STRING:string")
.expectLong( "A-INT:int", null)
.expectLong( "A-LONG:long", null)
.noLong( "A-FLOAT:float")
.noLong( "A-DOUBLE:double")
.expectDouble( "A-ANY:any", null)
.noDouble( "A-STRING:string")
.noDouble( "A-INT:int")
.noDouble( "A-LONG:long")
.expectDouble( "A-FLOAT:float", null)
.expectDouble( "A-DOUBLE:double", null)
// Not Null
.expectString( "N-ANY:any", "")
.expectString( "N-STRING:string", "")
.expectString( "N-INT:int", "")
.expectString( "N-LONG:long", "")
.expectString( "N-FLOAT:float", "")
.expectString( "N-DOUBLE:double", "")
.noLong( "N-ANY:any")
.noLong( "N-STRING:string")
.noLong( "N-INT:int")
.noLong( "N-LONG:long")
.noLong( "N-FLOAT:float")
.noLong( "N-DOUBLE:double")
.noDouble( "N-ANY:any")
.noDouble( "N-STRING:string")
.noDouble( "N-INT:int")
.noDouble( "N-LONG:long")
.noDouble( "N-FLOAT:float")
.noDouble( "N-DOUBLE:double")
// Not Empty
.noString( "E-ANY:any")
.noString( "E-STRING:string")
.noString( "E-INT:int")
.noString( "E-LONG:long")
.noString( "E-FLOAT:float")
.noString( "E-DOUBLE:double")
.noLong( "E-ANY:any")
.noLong( "E-STRING:string")
.noLong( "E-INT:int")
.noLong( "E-LONG:long")
.noLong( "E-FLOAT:float")
.noLong( "E-DOUBLE:double")
.noDouble( "E-ANY:any")
.noDouble( "E-STRING:string")
.noDouble( "E-INT:int")
.noDouble( "E-LONG:long")
.noDouble( "E-FLOAT:float")
.noDouble( "E-DOUBLE:double");
}
Example 15
| Source File: TestFieldSetters.java From logparser with Apache License 2.0 | 4 votes |
|
@Test
public void testNullValues() throws InvalidDissectorException, MissingDissectorsException, DissectionFailure {
Parser<TestFieldSettersRecord> parser = new Parser<>(TestFieldSettersRecord.class);
parser.setRootType("INPUT");
parser.addDissector(new NullValuesDissector());
TestFieldSettersRecord testRecord = parser.parse("Doesn't matter");
testRecord
// Default (== Always)
.expectString( "D-ANY:any", null)
.expectString( "D-STRING:string", null)
.expectString( "D-INT:int", null)
.expectString( "D-LONG:long", null)
.expectString( "D-FLOAT:float", null)
.expectString( "D-DOUBLE:double", null)
.expectLong( "D-ANY:any", null)
.noLong( "D-STRING:string")
.expectLong( "D-INT:int", null)
.expectLong( "D-LONG:long", null)
.noLong( "D-FLOAT:float")
.noLong( "D-DOUBLE:double")
.expectDouble( "D-ANY:any", null)
.noDouble( "D-STRING:string")
.noDouble( "D-INT:int")
.noDouble( "D-LONG:long")
.expectDouble( "D-FLOAT:float", null)
.expectDouble( "D-DOUBLE:double", null)
// Always
.expectString( "A-ANY:any", null)
.expectString( "A-STRING:string", null)
.expectString( "A-INT:int", null)
.expectString( "A-LONG:long", null)
.expectString( "A-FLOAT:float", null)
.expectString( "A-DOUBLE:double", null)
.expectLong( "A-ANY:any", null)
.noLong( "A-STRING:string")
.expectLong( "A-INT:int", null)
.expectLong( "A-LONG:long", null)
.noLong( "A-FLOAT:float")
.noLong( "A-DOUBLE:double")
.expectDouble( "A-ANY:any", null)
.noDouble( "A-STRING:string")
.noDouble( "A-INT:int")
.noDouble( "A-LONG:long")
.expectDouble( "A-FLOAT:float", null)
.expectDouble( "A-DOUBLE:double", null)
// Not Null
.noString( "N-ANY:any")
.noString( "N-STRING:string")
.noString( "N-INT:int")
.noString( "N-LONG:long")
.noString( "N-FLOAT:float")
.noString( "N-DOUBLE:double")
.noLong( "N-ANY:any")
.noLong( "N-STRING:string")
.noLong( "N-INT:int")
.noLong( "N-LONG:long")
.noLong( "N-FLOAT:float")
.noLong( "N-DOUBLE:double")
.noDouble( "N-ANY:any")
.noDouble( "N-STRING:string")
.noDouble( "N-INT:int")
.noDouble( "N-LONG:long")
.noDouble( "N-FLOAT:float")
.noDouble( "N-DOUBLE:double")
// Not Empty
.noString( "E-ANY:any")
.noString( "E-STRING:string")
.noString( "E-INT:int")
.noString( "E-LONG:long")
.noString( "E-FLOAT:float")
.noString( "E-DOUBLE:double")
.noLong( "E-ANY:any")
.noLong( "E-STRING:string")
.noLong( "E-INT:int")
.noLong( "E-LONG:long")
.noLong( "E-FLOAT:float")
.noLong( "E-DOUBLE:double")
.noDouble( "E-ANY:any")
.noDouble( "E-STRING:string")
.noDouble( "E-INT:int")
.noDouble( "E-LONG:long")
.noDouble( "E-FLOAT:float")
.noDouble( "E-DOUBLE:double");
}
