Java Code Examples for nl.basjes.parse.core.Parser#addDissector()
The following examples show how to use
nl.basjes.parse.core.Parser#addDissector() .
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: TestCase.java From logparser with Apache License 2.0 | 6 votes |
public static Parser<TestRecord> createTestParser() throws NoSuchMethodException { Parser<TestRecord> parser = new HttpdLoglineParser<>(TestRecord.class, getLogFormat()); parser.addDissector(new ScreenResolutionDissector()); parser.addTypeRemapping("request.firstline.uri.query.g", "HTTP.URI"); parser.addTypeRemapping("request.firstline.uri.query.r", "HTTP.URI"); parser.addTypeRemapping("request.firstline.uri.query.s", "SCREENRESOLUTION"); parser.addParseTarget("setConnectionClientHost", "IP:connection.client.host"); parser.addParseTarget("setRequestReceiveTime", "TIME.STAMP:request.receive.time"); parser.addParseTarget("setReferrer", "STRING:request.firstline.uri.query.g.query.promo"); parser.addParseTarget("setScreenResolution", "STRING:request.firstline.uri.query.s"); parser.addParseTarget("setScreenWidth", "SCREENWIDTH:request.firstline.uri.query.s.width"); parser.addParseTarget("setScreenHeight", "SCREENHEIGHT:request.firstline.uri.query.s.height"); parser.addParseTarget("setGoogleQuery", "STRING:request.firstline.uri.query.r.query.blabla"); parser.addParseTarget("setBui", "HTTP.COOKIE:request.cookies.bui"); parser.addParseTarget("setUseragent", "HTTP.USERAGENT:request.user-agent"); return parser; }
Example 2
Source File: TestDissectUserAgent.java From yauaa with Apache License 2.0 | 5 votes |
@Test public void testExtractUrlFields() { Parser<TestRecord> parser = new HttpdLoglineParser<>(TestRecord.class, "%t \"%{User-agent}i\""); parser.addDissector(new UserAgentDissector()); String testUri = "https://yauaa.basjes.nl:8080/something.html?aap=noot&mies=wim#zus"; String testUserAgent = "Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) " + "AppleWebKit/537.36 (KHTML, like Gecko) " + "Chrome/41.0.2272.96 " + "Mobile Safari/537.36" + "(" + testUri + ")"; String testLogLine = "[10/Aug/2012:23:55:11 +0200] \""+testUserAgent+"\""; DissectorTester .create() .withParser(parser) .withInput(testLogLine) // Did we get the field .expect("HTTP.USERAGENT:request.user-agent", testUserAgent) // Basic dissections .expect("STRING:request.user-agent.device_class", "Phone") .expect("STRING:request.user-agent.agent_name_version", "Chrome 41.0.2272.96") .expect("HTTP.URI:request.user-agent.agent_information_url", testUri) // Further extractions from the URI we found .expect("HTTP.PROTOCOL:request.user-agent.agent_information_url.protocol", "https") .expect("HTTP.HOST:request.user-agent.agent_information_url.host", "yauaa.basjes.nl") .expect("HTTP.PORT:request.user-agent.agent_information_url.port", "8080") .expect("HTTP.PATH:request.user-agent.agent_information_url.path", "/something.html") .expect("HTTP.QUERYSTRING:request.user-agent.agent_information_url.query", "&aap=noot&mies=wim") .expect("STRING:request.user-agent.agent_information_url.query.aap", "noot") .expect("STRING:request.user-agent.agent_information_url.query.mies", "wim") .expect("HTTP.REF:request.user-agent.agent_information_url.ref", "zus") .checkExpectations(); }
Example 3
Source File: TestCase.java From logparser with Apache License 2.0 | 5 votes |
public static Parser<TestRecord> createTestParser() throws NoSuchMethodException { Parser<TestRecord> parser = new HttpdLoglineParser<>(TestRecord.class, getLogFormat()); parser.addDissector(new nl.basjes.parse.httpdlog.dissectors.ScreenResolutionDissector()); parser.addTypeRemapping("request.firstline.uri.query.g", "HTTP.URI"); parser.addTypeRemapping("request.firstline.uri.query.r", "HTTP.URI"); parser.addTypeRemapping("request.firstline.uri.query.s", "SCREENRESOLUTION"); parser.addParseTarget("setConnectionClientHost", "IP:connection.client.host"); parser.addParseTarget("setRequestReceiveTime", "TIME.STAMP:request.receive.time"); parser.addParseTarget("setReferrer", "STRING:request.firstline.uri.query.g.query.promo"); parser.addParseTarget("setScreenResolution", "STRING:request.firstline.uri.query.s"); parser.addParseTarget("setScreenWidth", "SCREENWIDTH:request.firstline.uri.query.s.width"); parser.addParseTarget("setScreenHeight", "SCREENHEIGHT:request.firstline.uri.query.s.height"); parser.addParseTarget("setGoogleQuery", "STRING:request.firstline.uri.query.r.query.blabla"); parser.addParseTarget("setBui", "HTTP.COOKIE:request.cookies.bui"); parser.addParseTarget("setUseragent", "HTTP.USERAGENT:request.user-agent"); parser.addDissector(new GeoIPISPDissector(ISP_TEST_MMDB)); parser.addParseTarget("setAsnNumber", "ASN:connection.client.host.asn.number"); parser.addParseTarget("setAsnOrganization", "STRING:connection.client.host.asn.organization"); parser.addParseTarget("setIspName", "STRING:connection.client.host.isp.name"); parser.addParseTarget("setIspOrganization", "STRING:connection.client.host.isp.organization"); parser.addDissector(new GeoIPCityDissector(CITY_TEST_MMDB)); parser.addParseTarget("setContinentName", "STRING:connection.client.host.continent.name"); parser.addParseTarget("setContinentCode", "STRING:connection.client.host.continent.code"); parser.addParseTarget("setCountryName", "STRING:connection.client.host.country.name"); parser.addParseTarget("setCountryIso", "STRING:connection.client.host.country.iso"); parser.addParseTarget("setSubdivisionName", "STRING:connection.client.host.subdivision.name"); parser.addParseTarget("setSubdivisionIso", "STRING:connection.client.host.subdivision.iso"); parser.addParseTarget("setCityName", "STRING:connection.client.host.city.name"); parser.addParseTarget("setPostalCode", "STRING:connection.client.host.postal.code"); parser.addParseTarget("setLocationLatitude", "STRING:connection.client.host.location.latitude"); parser.addParseTarget("setLocationLongitude", "STRING:connection.client.host.location.longitude"); return parser; }
Example 4
Source File: TestCase.java From logparser with Apache License 2.0 | 5 votes |
public static Parser<TestRecord> createTestParser() throws NoSuchMethodException { Parser<TestRecord> parser = new HttpdLoglineParser<>(TestRecord.class, getLogFormat()); parser.addDissector(new nl.basjes.parse.httpdlog.dissectors.ScreenResolutionDissector()); parser.addTypeRemapping("request.firstline.uri.query.g", "HTTP.URI"); parser.addTypeRemapping("request.firstline.uri.query.r", "HTTP.URI"); parser.addTypeRemapping("request.firstline.uri.query.s", "SCREENRESOLUTION"); parser.addParseTarget("setConnectionClientHost", "IP:connection.client.host"); parser.addParseTarget("setRequestReceiveTime", "TIME.STAMP:request.receive.time"); parser.addParseTarget("setReferrer", "STRING:request.firstline.uri.query.g.query.promo"); parser.addParseTarget("setScreenResolution", "STRING:request.firstline.uri.query.s"); parser.addParseTarget("setScreenWidth", "SCREENWIDTH:request.firstline.uri.query.s.width"); parser.addParseTarget("setScreenHeight", "SCREENHEIGHT:request.firstline.uri.query.s.height"); parser.addParseTarget("setGoogleQuery", "STRING:request.firstline.uri.query.r.query.blabla"); parser.addParseTarget("setBui", "HTTP.COOKIE:request.cookies.bui"); parser.addParseTarget("setUseragent", "HTTP.USERAGENT:request.user-agent"); parser.addDissector(new GeoIPISPDissector(ISP_TEST_MMDB)); parser.addParseTarget("setAsnNumber", "ASN:connection.client.host.asn.number"); parser.addParseTarget("setAsnOrganization", "STRING:connection.client.host.asn.organization"); parser.addParseTarget("setIspName", "STRING:connection.client.host.isp.name"); parser.addParseTarget("setIspOrganization", "STRING:connection.client.host.isp.organization"); parser.addDissector(new GeoIPCityDissector(CITY_TEST_MMDB)); parser.addParseTarget("setContinentName", "STRING:connection.client.host.continent.name"); parser.addParseTarget("setContinentCode", "STRING:connection.client.host.continent.code"); parser.addParseTarget("setCountryName", "STRING:connection.client.host.country.name"); parser.addParseTarget("setCountryIso", "STRING:connection.client.host.country.iso"); parser.addParseTarget("setSubdivisionName", "STRING:connection.client.host.subdivision.name"); parser.addParseTarget("setSubdivisionIso", "STRING:connection.client.host.subdivision.iso"); parser.addParseTarget("setCityName", "STRING:connection.client.host.city.name"); parser.addParseTarget("setPostalCode", "STRING:connection.client.host.postal.code"); parser.addParseTarget("setLocationLatitude", "STRING:connection.client.host.location.latitude"); parser.addParseTarget("setLocationLongitude", "STRING:connection.client.host.location.longitude"); return parser; }
Example 5
Source File: NginxHttpdLogFormatDissector.java From logparser with Apache License 2.0 | 5 votes |
@Override public <RECORD> void createAdditionalDissectors(Parser<RECORD> parser) { super.createAdditionalDissectors(parser); parser.addDissector(new BinaryIPDissector()); parser.addDissector(new ConvertSecondsWithMillisStringDissector("SECOND_MILLIS", "MILLISECONDS")); parser.addDissector(new ConvertSecondsWithMillisStringDissector("TIME.EPOCH_SECOND_MILLIS", "TIME.EPOCH")); parser.addDissector(new ConvertMillisecondsIntoMicroseconds("MILLISECONDS", "MICROSECONDS")); modules.forEach(m -> parser.addDissectors(m.getDissectors())); }
Example 6
Source File: TokenFormatDissector.java From logparser with Apache License 2.0 | 4 votes |
@Override public <RECORD> void createAdditionalDissectors(Parser<RECORD> parser) { for (Token token: logFormatTokens) { parser.addDissector(token.getCustomDissector()); } }
Example 7
Source File: StrfTimeStampDissector.java From logparser with Apache License 2.0 | 4 votes |
@Override public <RECORD> void createAdditionalDissectors(Parser<RECORD> parser) { parser.addDissector(new LocalizedTimeDissector(inputType)); }
Example 8
Source File: ApacheHttpdLogParserTest.java From logparser with Apache License 2.0 | 4 votes |
/** * Test of initialize method, of class ApacheHttpdLogParser. */ @Test public void fullTest1() throws Exception { String line = "%127.0.0.1 127.0.0.1 127.0.0.1 - - [31/Dec/2012:23:49:40 +0100] " + "\"GET /icons/powered_by_rh.png?aap=noot&res=1024x768 HTTP/1.1\" 200 1213 " + "80 \"\" \"http://localhost/index.php?mies=wim\" 351 " + "\"Mozilla/5.0 (X11; Linux i686 on x86_64; rv:11.0) Gecko/20100101 Firefox/11.0\" " + "\"jquery-ui-theme=Eggplant\" \"Apache=127.0.0.1.1344635380111339; path=/; domain=.basjes.nl\" \"-\" " + "\"\\\"3780ff-4bd-4c1ce3df91380\\\"\""; Parser<TestRecord> parser = new HttpdLoglineParser<>(TestRecord.class, LOG_FORMAT); // Manually add an extra dissector parser.addDissector(new ScreenResolutionDissector()); parser.addTypeRemapping("request.firstline.uri.query.res", "SCREENRESOLUTION"); List<String> extraFields = new ArrayList<>(); extraFields.add("SCREENWIDTH:request.firstline.uri.query.res.width"); extraFields.add("SCREENHEIGHT:request.firstline.uri.query.res.height"); parser.addParseTarget(TestRecord.class.getMethod("setValue", String.class, String.class), extraFields); TestRecord record = new TestRecord(); parser.parse(record, line); Map<String, String> results = record.getResults(); System.out.println(results.toString()); assertEquals("noot", results.get("STRING:request.firstline.uri.query.aap")); assertEquals(null, results.get("STRING:request.firstline.uri.query.foo")); assertEquals(null, results.get("STRING:request.querystring.aap")); assertEquals("1024", results.get("SCREENWIDTH:request.firstline.uri.query.res.width")); assertEquals("768", results.get("SCREENHEIGHT:request.firstline.uri.query.res.height")); assertEquals("127.0.0.1", results.get("IP:connection.client.ip")); assertEquals(null, results.get("NUMBER:connection.client.logname")); assertEquals(null, results.get("STRING:connection.client.user")); assertEquals("31/Dec/2012:23:49:40 +0100", results.get("TIME.STAMP:request.receive.time")); assertEquals("1356994180000", results.get("TIME.EPOCH:request.receive.time.epoch")); assertEquals("1", results.get("TIME.WEEK:request.receive.time.weekofweekyear")); assertEquals("2013", results.get("TIME.YEAR:request.receive.time.weekyear")); assertEquals("2012", results.get("TIME.YEAR:request.receive.time.year")); assertEquals("40", results.get("TIME.SECOND:request.receive.time.second")); assertEquals("/icons/powered_by_rh.png?aap=noot&res=1024x768", results.get("HTTP.URI:request.firstline.uri")); assertEquals("200", results.get("STRING:request.status.last")); assertEquals("1213", results.get("BYTESCLF:response.body.bytes")); assertEquals("http://localhost/index.php?mies=wim", results.get("HTTP.URI:request.referer")); assertEquals("wim", results.get("STRING:request.referer.query.mies")); assertEquals("Mozilla/5.0 (X11; Linux i686 on x86_64; rv:11.0) Gecko/20100101 Firefox/11.0", results.get("HTTP.USERAGENT:request.user-agent")); assertEquals("31", results.get("TIME.DAY:request.receive.time.day")); assertEquals("23", results.get("TIME.HOUR:request.receive.time.hour")); assertEquals("December", results.get("TIME.MONTHNAME:request.receive.time.monthname")); assertEquals("351", results.get("MICROSECONDS:response.server.processing.time")); assertEquals("Apache=127.0.0.1.1344635380111339; path=/; domain=.basjes.nl", results.get("HTTP.SETCOOKIES:response.cookies")); assertEquals("jquery-ui-theme=Eggplant", results.get("HTTP.COOKIES:request.cookies")); assertEquals("\\\"3780ff-4bd-4c1ce3df91380\\\"", results.get("HTTP.HEADER:response.header.etag")); assertEquals("Eggplant", results.get("HTTP.COOKIE:request.cookies.jquery-ui-theme")); assertEquals("Apache=127.0.0.1.1344635380111339; path=/; domain=.basjes.nl", results.get("HTTP.SETCOOKIE:response.cookies.apache")); assertEquals(".basjes.nl", results.get("STRING:response.cookies.apache.domain")); }
Example 9
Source File: TestFieldSetters.java From logparser with Apache License 2.0 | 4 votes |
@Test public void testEmptyValues() throws InvalidDissectorException, MissingDissectorsException, DissectionFailure { Parser<TestFieldSettersRecord> parser = new Parser<>(TestFieldSettersRecord.class); parser.setRootType("INPUT"); parser.addDissector(new EmptyValuesDissector()); TestFieldSettersRecord testRecord = parser.parse("Doesn't matter"); testRecord // Default (== Always) .expectString( "D-ANY:any", "") .expectString( "D-STRING:string", "") .expectString( "D-INT:int", "") .expectString( "D-LONG:long", "") .expectString( "D-FLOAT:float", "") .expectString( "D-DOUBLE:double", "") .expectLong( "D-ANY:any", null) .noLong( "D-STRING:string") .expectLong( "D-INT:int", null) .expectLong( "D-LONG:long", null) .noLong( "D-FLOAT:float") .noLong( "D-DOUBLE:double") .expectDouble( "D-ANY:any", null) .noDouble( "D-STRING:string") .noDouble( "D-INT:int") .noDouble( "D-LONG:long") .expectDouble( "D-FLOAT:float", null) .expectDouble( "D-DOUBLE:double", null) // Always .expectString( "A-ANY:any", "") .expectString( "A-STRING:string", "") .expectString( "A-INT:int", "") .expectString( "A-LONG:long", "") .expectString( "A-FLOAT:float", "") .expectString( "A-DOUBLE:double", "") .expectLong( "A-ANY:any", null) .noLong( "A-STRING:string") .expectLong( "A-INT:int", null) .expectLong( "A-LONG:long", null) .noLong( "A-FLOAT:float") .noLong( "A-DOUBLE:double") .expectDouble( "A-ANY:any", null) .noDouble( "A-STRING:string") .noDouble( "A-INT:int") .noDouble( "A-LONG:long") .expectDouble( "A-FLOAT:float", null) .expectDouble( "A-DOUBLE:double", null) // Not Null .expectString( "N-ANY:any", "") .expectString( "N-STRING:string", "") .expectString( "N-INT:int", "") .expectString( "N-LONG:long", "") .expectString( "N-FLOAT:float", "") .expectString( "N-DOUBLE:double", "") .noLong( "N-ANY:any") .noLong( "N-STRING:string") .noLong( "N-INT:int") .noLong( "N-LONG:long") .noLong( "N-FLOAT:float") .noLong( "N-DOUBLE:double") .noDouble( "N-ANY:any") .noDouble( "N-STRING:string") .noDouble( "N-INT:int") .noDouble( "N-LONG:long") .noDouble( "N-FLOAT:float") .noDouble( "N-DOUBLE:double") // Not Empty .noString( "E-ANY:any") .noString( "E-STRING:string") .noString( "E-INT:int") .noString( "E-LONG:long") .noString( "E-FLOAT:float") .noString( "E-DOUBLE:double") .noLong( "E-ANY:any") .noLong( "E-STRING:string") .noLong( "E-INT:int") .noLong( "E-LONG:long") .noLong( "E-FLOAT:float") .noLong( "E-DOUBLE:double") .noDouble( "E-ANY:any") .noDouble( "E-STRING:string") .noDouble( "E-INT:int") .noDouble( "E-LONG:long") .noDouble( "E-FLOAT:float") .noDouble( "E-DOUBLE:double"); }
Example 10
Source File: TestFieldSetters.java From logparser with Apache License 2.0 | 4 votes |
@Test public void testNullValues() throws InvalidDissectorException, MissingDissectorsException, DissectionFailure { Parser<TestFieldSettersRecord> parser = new Parser<>(TestFieldSettersRecord.class); parser.setRootType("INPUT"); parser.addDissector(new NullValuesDissector()); TestFieldSettersRecord testRecord = parser.parse("Doesn't matter"); testRecord // Default (== Always) .expectString( "D-ANY:any", null) .expectString( "D-STRING:string", null) .expectString( "D-INT:int", null) .expectString( "D-LONG:long", null) .expectString( "D-FLOAT:float", null) .expectString( "D-DOUBLE:double", null) .expectLong( "D-ANY:any", null) .noLong( "D-STRING:string") .expectLong( "D-INT:int", null) .expectLong( "D-LONG:long", null) .noLong( "D-FLOAT:float") .noLong( "D-DOUBLE:double") .expectDouble( "D-ANY:any", null) .noDouble( "D-STRING:string") .noDouble( "D-INT:int") .noDouble( "D-LONG:long") .expectDouble( "D-FLOAT:float", null) .expectDouble( "D-DOUBLE:double", null) // Always .expectString( "A-ANY:any", null) .expectString( "A-STRING:string", null) .expectString( "A-INT:int", null) .expectString( "A-LONG:long", null) .expectString( "A-FLOAT:float", null) .expectString( "A-DOUBLE:double", null) .expectLong( "A-ANY:any", null) .noLong( "A-STRING:string") .expectLong( "A-INT:int", null) .expectLong( "A-LONG:long", null) .noLong( "A-FLOAT:float") .noLong( "A-DOUBLE:double") .expectDouble( "A-ANY:any", null) .noDouble( "A-STRING:string") .noDouble( "A-INT:int") .noDouble( "A-LONG:long") .expectDouble( "A-FLOAT:float", null) .expectDouble( "A-DOUBLE:double", null) // Not Null .noString( "N-ANY:any") .noString( "N-STRING:string") .noString( "N-INT:int") .noString( "N-LONG:long") .noString( "N-FLOAT:float") .noString( "N-DOUBLE:double") .noLong( "N-ANY:any") .noLong( "N-STRING:string") .noLong( "N-INT:int") .noLong( "N-LONG:long") .noLong( "N-FLOAT:float") .noLong( "N-DOUBLE:double") .noDouble( "N-ANY:any") .noDouble( "N-STRING:string") .noDouble( "N-INT:int") .noDouble( "N-LONG:long") .noDouble( "N-FLOAT:float") .noDouble( "N-DOUBLE:double") // Not Empty .noString( "E-ANY:any") .noString( "E-STRING:string") .noString( "E-INT:int") .noString( "E-LONG:long") .noString( "E-FLOAT:float") .noString( "E-DOUBLE:double") .noLong( "E-ANY:any") .noLong( "E-STRING:string") .noLong( "E-INT:int") .noLong( "E-LONG:long") .noLong( "E-FLOAT:float") .noLong( "E-DOUBLE:double") .noDouble( "E-ANY:any") .noDouble( "E-STRING:string") .noDouble( "E-INT:int") .noDouble( "E-LONG:long") .noDouble( "E-FLOAT:float") .noDouble( "E-DOUBLE:double"); }
Example 11
Source File: ReferenceTest.java From logparser with Apache License 2.0 | 4 votes |
@Test public void runManuallyCombined(){ Parser<TestRecord> parser = new Parser<>(TestRecord.class); parser.addDissector(new FooDissector()); parser.addDissector(new BarDissector()); parser.addTypeRemapping("foostring", "BARINPUT"); parser.setRootType(new FooDissector().getInputType()); DissectorTester.create() .withParser(parser) .withInput("BlaBlaBla") .expect("ANY:fooany", "42") .expect("ANY:fooany", 42L) .expect("ANY:fooany", 42D) .expect("STRING:foostring", "42") .expectAbsentLong("STRING:foostring") .expectAbsentDouble("STRING:foostring") .expect("INT:fooint", "42") .expect("INT:fooint", 42L) .expectAbsentDouble("INT:fooint") .expect("LONG:foolong", "42") .expect("LONG:foolong", 42L) .expectAbsentDouble("LONG:foolong") .expect("FLOAT:foofloat", "42.0") .expectAbsentLong("FLOAT:foofloat") .expect("FLOAT:foofloat", 42D) .expect("DOUBLE:foodouble", "42.0") .expectAbsentLong("DOUBLE:foodouble") .expect("DOUBLE:foodouble", 42D) .expect("ANY:foostring.barany", "42") .expect("ANY:foostring.barany", 42L) .expect("ANY:foostring.barany", 42D) .expect("STRING:foostring.barstring", "42") .expectAbsentLong("STRING:foostring.barstring") .expectAbsentDouble("STRING:foostring.barstring") .expect("INT:foostring.barint", "42") .expect("INT:foostring.barint", 42L) .expectAbsentDouble("INT:foostring.barint") .expect("LONG:foostring.barlong", "42") .expect("LONG:foostring.barlong", 42L) .expectAbsentDouble("LONG:foostring.barlong") .expect("FLOAT:foostring.barfloat", "42.0") .expectAbsentLong("FLOAT:foostring.barfloat") .expect("FLOAT:foostring.barfloat", 42D) .expect("DOUBLE:foostring.bardouble", "42.0") .expectAbsentLong("DOUBLE:foostring.bardouble") .expect("DOUBLE:foostring.bardouble", 42D) .checkExpectations(); }
Example 12
Source File: FooSpecialDissector.java From logparser with Apache License 2.0 | 4 votes |
@Override public <RECORD> void createAdditionalDissectors(Parser<RECORD> parser) { parser.addDissector(new BarDissector()); parser.addTypeRemapping("foostring", "BARINPUT"); }