nl.basjes.parse.useragent.UserAgentAnalyzer Java Examples

@Test
public void testLoadMoreResources() {
    UserAgentAnalyzerBuilder builder =
        UserAgentAnalyzer.newBuilder().delayInitialization().withField("DeviceClass");

    UserAgentAnalyzer uaa = builder.build();
    assertNotNull(uaa, "We should get a first instance from a single builder.");

    // Try to load something that does not yield any files.
    // Optional --> error message and continue
    uaa.loadResources("Bad resource string that is optional should only give a warning", true, true);

    // NOT Optional --> fail with exception
    assertThrows(InvalidParserConfigurationException.class, () ->
        uaa.loadResources("Bad resource string that is NOT optional should fail hard", true, false)
    );

    uaa.initializeMatchers();
    assertThrows(IllegalStateException.class, () ->
        uaa.loadResources("Cannot load resources after initialization")
    );
}
 

@Test
public void testPostPreheatDroptests() {
    UserAgentAnalyzer userAgentAnalyzer =
        UserAgentAnalyzer
            .newBuilder()
            .immediateInitialization()
            // Without .preheat(100)
            .dropTests()
            .hideMatcherLoadStats()
            .withField("AgentName")
            .build();
    assertEquals(0, userAgentAnalyzer.getNumberOfTestCases());

    userAgentAnalyzer =
        UserAgentAnalyzer
            .newBuilder()
            .immediateInitialization()
            .preheat(100) // With .preheat(100)
            .dropTests()
            .hideMatcherLoadStats()
            .withField("AgentName")
            .build();
    assertEquals(0, userAgentAnalyzer.getNumberOfTestCases());
}
 

@Test
public void testLoadOnlyCompanyCustomFormatRules() {
    UserAgentAnalyzer userAgentAnalyzer =
        UserAgentAnalyzer
            .newBuilder()
            .withoutCache()
            .hideMatcherLoadStats()
            .dropDefaultResources()
            .addResources("CompanyInternalUserAgents.yaml")
            .withFields("ApplicationName", "ApplicationVersion")
            .withFields(Arrays.asList("ApplicationInstance", "ApplicationGitCommit"))
            .withField("ServerName")
            .build();

    UserAgent parsedAgent = userAgentAnalyzer.parse(
        "TestApplication/1.2.3 (node123.datacenter.example.nl; 1234; d71922715c2bfe29343644b14a4731bf5690e66e)");

    // The requested fields
    assertEquals("TestApplication",                          parsedAgent.getValue("ApplicationName"));
    assertEquals("1.2.3",                                    parsedAgent.getValue("ApplicationVersion"));
    assertEquals("1234",                                     parsedAgent.getValue("ApplicationInstance"));
    assertEquals("d71922715c2bfe29343644b14a4731bf5690e66e", parsedAgent.getValue("ApplicationGitCommit"));
    assertEquals("node123.datacenter.example.nl",            parsedAgent.getValue("ServerName"));
}
 

@Override
public List<String> getPossibleOutput() {
    List<String> result = new ArrayList<>();

    // First the standard fields in the standard order, then the non-standard fields alphabetically
    final UserAgentAnalyzerBuilder builder = UserAgentAnalyzer.newBuilder();
    extraResources.forEach(builder::addResources);

    allPossibleFieldNames = builder.build().getAllPossibleFieldNamesSorted();
    for (String fieldName : allPossibleFieldNames) {
        ensureMappingsExistForFieldName(fieldName);
        result.add(getFieldOutputType(fieldName) + ":" + fieldNameToDissectionName(fieldName));
    }

    return result;
}
 

@Test
public void testLoadOnlyCustomRules() {
    UserAgentAnalyzer userAgentAnalyzer =
        UserAgentAnalyzer
            .newBuilder()
            .withoutCache()
            .hideMatcherLoadStats()
            .addResources("ExtraLoadedRule1.yaml")
            .withField("ExtraValue2")
            .withField("ExtraValue1")
            .addResources("ExtraLoadedRule2.yaml")
            .build();

    UserAgent parsedAgent = userAgentAnalyzer.parse("Mozilla/5.0 (Linux; Android 7.0; Nexus 6 Build/NBD90Z) " +
        "AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2785.124 Mobile Safari/537.36");

    // The requested fields
    assertEquals("One",    parsedAgent.getValue("ExtraValue1" ));
    assertEquals("Two",    parsedAgent.getValue("ExtraValue2" ));
}
 

private void initialize() {
    if (!initialized) {
        UserAgentAnalyzerBuilder analyzerBuilder = UserAgentAnalyzer
                .newBuilder()
                .hideMatcherLoadStats()
                .delayInitialization();

        if (cacheSize >= 0) {
            analyzerBuilder.withCache(cacheSize);
        }

        if (!requestedFields.isEmpty()) {
            for (String requestedField : requestedFields) {
                analyzerBuilder.withField(requestedField);
            }
        }

        analyzer = analyzerBuilder.build();

        if (requestedFields.isEmpty()) {
            requestedFields.addAll(analyzer.getAllPossibleFieldNamesSorted());
        }

        initialized = true;
    }
}
 

@Disabled
    @Test
    public void checkForMemoryLeaksDuringRuns() { //NOSONAR: Do not complain about ignored performance test
        UserAgentAnalyzer uaa = UserAgentAnalyzer
            .newBuilder()
            .withoutCache()
//            .withField("OperatingSystemName")
//            .withField("OperatingSystemVersion")
//            .withField("DeviceClass")
            .hideMatcherLoadStats()
            .keepTests()
            .build();

        LOG.info("Init complete");
        int iterationsDone = 0;
        final int iterationsPerLoop = 1000;
        for (int i = 0; i < 100; i++) {
            long start = System.nanoTime();
            uaa.preHeat(iterationsPerLoop, false);
            long stop = System.nanoTime();
            iterationsDone += iterationsPerLoop;

            long averageNanos = (stop - start) / iterationsPerLoop;
            printMemoryUsage(iterationsDone, averageNanos);
        }
    }
 

@Test
public void testLoadAdditionalRules() {
    UserAgentAnalyzer userAgentAnalyzer =
        UserAgentAnalyzer
            .newBuilder()
            .withField("DeviceClass")
            .withoutCache()
            .hideMatcherLoadStats()
            .addResources("ExtraLoadedRule1.yaml")
            .withField("ExtraValue2")
            .withField("ExtraValue1")
            .addResources("ExtraLoadedRule2.yaml")
            .build();

    UserAgent parsedAgent = userAgentAnalyzer.parse("Mozilla/5.0 (Linux; Android 7.0; Nexus 6 Build/NBD90Z) " +
        "AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2785.124 Mobile Safari/537.36");

    // The requested fields
    assertEquals("Phone",  parsedAgent.getValue("DeviceClass" ));
    assertEquals("One",    parsedAgent.getValue("ExtraValue1" ));
    assertEquals("Two",    parsedAgent.getValue("ExtraValue2" ));
}
 

@SuppressWarnings("unused") // Called via the annotation
@OnScheduled
public void onSchedule(ProcessContext context) {
    if (uaa == null) {
        UserAgentAnalyzerBuilder builder =
            UserAgentAnalyzer
            .newBuilder()
            .hideMatcherLoadStats()
            .dropTests();

        extractFieldNames.clear();

        for (PropertyDescriptor propertyDescriptor: supportedPropertyDescriptors) {
            if (context.getProperty(propertyDescriptor).asBoolean()) {
                String name = propertyDescriptor.getName();
                if (name.startsWith(PROPERTY_PREFIX)) { // Should always pass
                    String fieldName = name.substring(PROPERTY_PREFIX.length());

                    builder.withField(fieldName);
                    extractFieldNames.add(fieldName);
                }
            }
        }
        uaa = builder.build();
    }
}
 

@Test
public void testLimitedFields() {
    UserAgentAnalyzer userAgentAnalyzer =
        UserAgentAnalyzer
            .newBuilder()
            .preheat(100)
            .preheat()
            .withCache(42)
            .withoutCache()
            .hideMatcherLoadStats()
            .showMatcherLoadStats()
            .withAllFields()
            .withField("DeviceClass")
            .withField("AgentNameVersionMajor")
            .withUserAgentMaxLength(1234)
            .build();

    assertEquals(1234, userAgentAnalyzer.getUserAgentMaxLength());

    runTestCase(userAgentAnalyzer);
}
 

@Test
public void testPreheatNoTests() {
    UserAgentAnalyzer userAgentAnalyzer =
        UserAgentAnalyzer
            .newBuilder()
            .keepTests()
            .hideMatcherLoadStats()
            .withField("AgentName")
            .build();

    assertTrue(userAgentAnalyzer.getNumberOfTestCases() > 100);
    assertEquals(0, userAgentAnalyzer.preHeat(0));
    assertEquals(0, userAgentAnalyzer.preHeat(-1));
    assertEquals(0, userAgentAnalyzer.preHeat(1000000000L));

    userAgentAnalyzer.dropTests();
    assertEquals(0, userAgentAnalyzer.getNumberOfTestCases());
    assertEquals(0, userAgentAnalyzer.preHeat());
}
 

@Test
public void testCacheSetter() {
    UserAgentAnalyzer userAgentAnalyzer = UserAgentAnalyzer.newBuilder().build();
    userAgentAnalyzer.loadResources("classpath*:AllFields-tests.yaml");

    assertEquals(10000, userAgentAnalyzer.getCacheSize(), "Incorrect default cache size");

    userAgentAnalyzer.setCacheSize(50);
    assertEquals(50, userAgentAnalyzer.getCacheSize(), "Incorrect default cache size");

    userAgentAnalyzer.setCacheSize(50000);
    assertEquals(50000, userAgentAnalyzer.getCacheSize(), "Incorrect default cache size");

    userAgentAnalyzer.setCacheSize(-5);
    assertEquals(0, userAgentAnalyzer.getCacheSize(), "Incorrect default cache size");

    userAgentAnalyzer.setCacheSize(50);
    assertEquals(50, userAgentAnalyzer.getCacheSize(), "Incorrect default cache size");

    userAgentAnalyzer.setCacheSize(50000);
    assertEquals(50000, userAgentAnalyzer.getCacheSize(), "Incorrect default cache size");

    userAgentAnalyzer.setUserAgentMaxLength(555);
    assertEquals(555, userAgentAnalyzer.getUserAgentMaxLength(), "Incorrect user agent max length");
}
 

@Test
public void runBenchmarks() {

    uaa = UserAgentAnalyzer
        .newBuilder()
        .withoutCache()
        .preheat(10000)
        .build();

    List<Triple<Counter, String, String>> testCases = createTestCasesList();

    System.gc(); // Avoid gc during tests

    for (int run = 1; run < 10000; run++) {
        if (run % 100 == 0) {
            System.gc(); // Avoid gc during tests
            LOG.info("Did {} runs", run);
        }
        testCases.forEach(this::doTest);
    }

    testCases.forEach(this::printResults);

}
 

@Override
public void open(FunctionContext context) {
    userAgentAnalyzer = UserAgentAnalyzer
        .newBuilder()
        .withFields(extractedFields)
        .withCache(cacheSize)
        .immediateInitialization()
        .build();

    if (extractedFields.isEmpty()) {
        extractedFields.addAll(userAgentAnalyzer.getAllPossibleFieldNamesSorted());
    }
}
 

public ThreadState() {
    uaa = UserAgentAnalyzer.newBuilder()
        .withoutCache()
        .hideMatcherLoadStats()
        .build();
    uaa.parse((String)null);
}
 

@Override
public ObjectInspector initialize(ObjectInspector[] args) throws UDFArgumentException {
    // ================================
    // Check the input
    // This UDF accepts one argument
    if (args.length != 1) {
        throw new UDFArgumentException("The argument list must be exactly 1 element");
    }

    // The first argument must be a String
    ObjectInspector inputOI = args[0];
    if (!(inputOI instanceof StringObjectInspector)) {
        throw new UDFArgumentException("The argument must be a string");
    }
    useragentOI = (StringObjectInspector) inputOI;

    // ================================
    // Initialize the parser
    userAgentAnalyzer = UserAgentAnalyzer
        .newBuilder()
        .hideMatcherLoadStats()
        .delayInitialization()
        .build();

    fieldNames = userAgentAnalyzer.getAllPossibleFieldNamesSorted();

    // ================================
    // Define the output
    // https://stackoverflow.com/questions/26026027/how-to-return-struct-from-hive-udf

    // Define the field names for the struct<> and their types
    List<ObjectInspector> fieldObjectInspectors = new ArrayList<>(fieldNames.size());

    fieldNames
        .forEach(f -> fieldObjectInspectors.add(PrimitiveObjectInspectorFactory.writableStringObjectInspector));

    return ObjectInspectorFactory.getStandardStructObjectInspector(fieldNames, fieldObjectInspectors);
}
 

@Override
    public Map<String, String> apply(String input) {
        if (userAgentAnalyzer == null) {
            userAgentAnalyzer = UserAgentAnalyzer
                .newBuilder()
                .immediateInitialization()
                .hideMatcherLoadStats()
                .withCache(cacheSize)
//                .withFields() // TODO: Implement this later
                .build();
        }

        return userAgentAnalyzer.parse(input).toMap();
    }
 

@Override
protected void init(ProcessorInitializationContext context) {
    super.init(context);

    synchronized (ALL_FIELD_NAMES) {
        if (ALL_FIELD_NAMES.isEmpty()) {
            ALL_FIELD_NAMES.addAll(UserAgentAnalyzer
                .newBuilder()
                .hideMatcherLoadStats()
                .delayInitialization()
                .dropTests()
                .build()
                .getAllPossibleFieldNamesSorted());
        }
    }
    final Set<Relationship> relationshipsSet = new HashSet<>();
    relationshipsSet.add(SUCCESS);
    relationshipsSet.add(MISSING);
    this.relationships = Collections.unmodifiableSet(relationshipsSet);

    for (String fieldName: ALL_FIELD_NAMES) {
        PropertyDescriptor propertyDescriptor = new PropertyDescriptor.Builder()
            .name(PROPERTY_PREFIX + fieldName)
            .description("If enabled will extract the " + fieldName + " field")
            .required(true)
            .allowableValues("true", "false")
            .defaultValue("false")
            .addValidator(StandardValidators.BOOLEAN_VALIDATOR)
            .build();
        supportedPropertyDescriptors.add(propertyDescriptor);
    }

}
 

public Yauaa(String id, Configuration config, Context context) {
    this.id = id;
    // constructors should validate configuration options
    sourceField = config.get(SOURCE_CONFIG);
    final Map<String, Object> requestedFields = config.get(FIELDS_CONFIG);

    if (requestedFields != null) {
        outputFields = new HashMap<>();
        requestedFields.forEach((key, value) -> outputFields.put(key, value.toString()));
    }

    checkConfiguration();

    UserAgentAnalyzerBuilder userAgentAnalyzerBuilder =
        UserAgentAnalyzer
            .newBuilder()
            .immediateInitialization()
            .dropTests()
            .hideMatcherLoadStats();

    outputFields.forEach((yauaaFieldName, outputFieldName) -> {
        requestedFieldNames.add(yauaaFieldName);
        userAgentAnalyzerBuilder.withField(yauaaFieldName);
    });

    userAgentAnalyzer = userAgentAnalyzerBuilder.build();
}
 

private UserAgentAnalyzerBuilder getUserAgentAnalyzerBuilder() {
    if (userAgentAnalyzerBuilder == null) {
        userAgentAnalyzerBuilder = UserAgentAnalyzer
            .newBuilder()
            .delayInitialization()
            .dropTests()
            .hideMatcherLoadStats();
    }
    return userAgentAnalyzerBuilder;
}
 

public static synchronized UserAgentAnalyzer getInstance() {
    if (instance == null) {
        instance = UserAgentAnalyzer
            .newBuilder()
            .dropTests()
            .hideMatcherLoadStats()
            .immediateInitialization()
            .build();
    }
    return instance;
}
 

public YauaaProcessor(String tag,
                      String field,
                      String targetField,
                      Collection<String> fieldNames,
                      Integer cacheSize,
                      Integer preheat,
                      String  extraRules) {
    super(tag);
    this.field = field;
    this.targetField = targetField;

    UserAgentAnalyzerBuilder builder = UserAgentAnalyzer
        .newBuilder()
        .dropTests()
        .immediateInitialization();

    if (cacheSize >= 0) {
        builder.withCache(cacheSize);
    }

    if (preheat >= 0) {
        builder.preheat(preheat);
    }

    if (extraRules != null) {
        builder.addYamlRule(extraRules);
    }

    if (fieldNames != null && !fieldNames.isEmpty()) {
        builder.withFields(fieldNames);
    }

    this.uaa = builder.build();
}
 

@PostConstruct
public void automaticStartup() {
    setInstance(this);

    if (!userAgentAnalyzerIsAvailable && userAgentAnalyzerFailureMessage == null) {
        initStartMoment = System.currentTimeMillis();
        new Thread(() -> {
            try {
                userAgentAnalyzer = UserAgentAnalyzer.newBuilder()
                    .hideMatcherLoadStats()
                    .addOptionalResources("file:UserAgents*/*.yaml")
                    .immediateInitialization()
                    .keepTests()
                    .build();
                userAgentAnalyzerIsAvailable = true;
            } catch (Exception e) {
                userAgentAnalyzerFailureMessage =
                    e.getClass().getSimpleName() + "<br/>" +
                        e.getMessage().replaceAll("\n", "<br/>");
                LOG.error("Fatal error during startup: {}\n" +
                        "=======================================================\n" +
                        "{}\n" +
                        "=======================================================\n",
                        e.getClass().getCanonicalName(), e.getMessage());
            }
        }).start();
    }
}
 

@Test
public void testBadVersion(){

    InvalidParserConfigurationException exception =

        assertThrows(InvalidParserConfigurationException.class, () ->
            UserAgentAnalyzer
        .newBuilder()
        .dropDefaultResources()
        .addResources("classpath*:Versions/BadVersion.yaml")
        .delayInitialization()
        .build());
    assertTrue(exception.getMessage().contains("Found unexpected config entry: bad"));
}
 

@PreDestroy
public void preDestroy() {
    if (userAgentAnalyzer != null) {
        UserAgentAnalyzer uaa = userAgentAnalyzer;
        // First we disable it for all uses.
        userAgentAnalyzer = null;
        userAgentAnalyzerIsAvailable = false;
        userAgentAnalyzerFailureMessage = "UserAgentAnalyzer has been destroyed.";
        // Then we actually wipe it.
        uaa.destroy();
    }
    setInstance(null);
}
 

public Demo() {
    uaa = UserAgentAnalyzer
        .newBuilder()
        .withCache(1234)
        .withField("DeviceClass")
        .withAllFields()
        .build();
}
 

@Test
public void testAnnotationCacheSetting2() throws IllegalAccessException, NoSuchFieldException {
    UserAgentAnnotationAnalyzer<TestRecord> userAgentAnnotationAnalyzer = new UserAgentAnnotationAnalyzer<>();

    // To make sure the internals behave as expected
    Field userAgentAnalyzerField = userAgentAnnotationAnalyzer.getClass().getDeclaredField("userAgentAnalyzer");
    userAgentAnalyzerField.setAccessible(true);
    assertNull(userAgentAnalyzerField.get(userAgentAnnotationAnalyzer));

    // Initial value
    assertEquals(DEFAULT_PARSE_CACHE_SIZE, userAgentAnnotationAnalyzer.getCacheSize());

    userAgentAnnotationAnalyzer.initialize(new MyMapper());

    UserAgentAnalyzer userAgentAnalyzer = (UserAgentAnalyzer)userAgentAnalyzerField.get(userAgentAnnotationAnalyzer);
    assertNotNull(userAgentAnalyzer);
    // Initial value
    assertEquals(DEFAULT_PARSE_CACHE_SIZE, userAgentAnnotationAnalyzer.getCacheSize());
    assertEquals(DEFAULT_PARSE_CACHE_SIZE, userAgentAnalyzer.getCacheSize());

    // Setting and getting while there IS a UserAgentAnalyzer
    userAgentAnnotationAnalyzer.setCacheSize(1234);
    assertEquals(1234, userAgentAnnotationAnalyzer.getCacheSize());
    assertEquals(1234, userAgentAnalyzer.getCacheSize());

    userAgentAnnotationAnalyzer.disableCaching();
    assertEquals(0, userAgentAnnotationAnalyzer.getCacheSize());
    assertEquals(0, userAgentAnalyzer.getCacheSize());

    userAgentAnnotationAnalyzer.setCacheSize(4567);
    assertEquals(4567, userAgentAnnotationAnalyzer.getCacheSize());
    assertEquals(4567, userAgentAnalyzer.getCacheSize());

}
 

@Test
    public void checkAllPossibleFieldsFastSpeed() {
        LOG.info("Create analyzer");
        long start = System.nanoTime();
        UserAgentAnalyzer uaa = UserAgentAnalyzer
            .newBuilder()
            .keepTests()
            .delayInitialization()
            .build();
        long stop = System.nanoTime();
        long constructMsecs = (stop - start) / 1000000;
        LOG.info("-- Construction time: {}ms", constructMsecs);

        LOG.info("List fieldnames");
        start = System.nanoTime();
        uaa.getAllPossibleFieldNamesSorted()
            .forEach(LOG::info);
        stop = System.nanoTime();
        long listFieldNamesMsecs = (stop - start) / 1000000;
        LOG.info("-- List fieldnames: {}ms", listFieldNamesMsecs);
        assertTrue(listFieldNamesMsecs < 500, "Just listing the field names should only take a few ms");

        LOG.info("Initializing the datastructures");
        start = System.nanoTime();
        uaa.initializeMatchers();
        stop = System.nanoTime();
        long initializeMsecs = (stop - start) / 1000000;
        LOG.info("-- Initialization: {}ms", initializeMsecs);
//        assertTrue("The initialization went too fast, this should take several seconds", initializeMsecs > 100);

        LOG.info("Preheat");
        start = System.nanoTime();
        uaa.preHeat();
        stop = System.nanoTime();
        long preheatMsecs = (stop - start) / 1000000;
        LOG.info("-- Preheat : {}ms", preheatMsecs);
    }
 

@Disabled
@Test
public void profileMemoryFootprint() { //NOSONAR: Do not complain about ignored performance test
    printCurrentMemoryProfile("Before ");

    UserAgentAnalyzer uaa = UserAgentAnalyzer
        .newBuilder()
        .hideMatcherLoadStats()
        .withoutCache()
        .keepTests()
        .build();
    printCurrentMemoryProfile("Loaded ");

    uaa.initializeMatchers();
    printCurrentMemoryProfile("Init   ");

    Runtime.getRuntime().gc();
    printCurrentMemoryProfile("Post GC");

    uaa.setCacheSize(1000);
    uaa.preHeat();
    Runtime.getRuntime().gc();
    printCurrentMemoryProfile("Cache 1K");

    uaa.setCacheSize(10000);
    uaa.preHeat();
    Runtime.getRuntime().gc();
    printCurrentMemoryProfile("Cache 10K");

    uaa.dropTests();
    Runtime.getRuntime().gc();
    printCurrentMemoryProfile("NoTest ");

}
 

@Test
public void testDualBuilderUsageUseSetterAfterBuild() {
    UserAgentAnalyzerBuilder builder =
        UserAgentAnalyzer.newBuilder().delayInitialization();

    assertNotNull(builder.build(), "We should get a first instance from a single builder.");

    // And calling a setter after the build() should fail with an exception
    assertThrows(IllegalStateException.class, () ->
        builder.withCache(1234)
    );
}