org.apache.commons.collections4.iterators.TransformIterator Java Examples

@SuppressWarnings("unchecked")
@Test
public void testConstructorSetsConnection() throws Exception {
    DatawaveUser user = new DatawaveUser(userDN, UserType.USER, null, null, null, 0L);
    DatawavePrincipal principal = new DatawavePrincipal(Collections.singletonList(user));
    
    // setup mock connector
    InMemoryInstance instance = new InMemoryInstance("test instance");
    Connector connector = instance.getConnector("root", new PasswordToken(""));
    
    // setup mock logic, handles the setConnection method
    SampleGenericQueryConfiguration config = new SampleGenericQueryConfiguration();
    expect(logic.initialize(anyObject(), anyObject(), anyObject())).andReturn(config);
    logic.setupQuery(config);
    TransformIterator iter = new TransformIterator();
    expect(logic.getCollectQueryMetrics()).andReturn(Boolean.FALSE);
    expect(logic.getTransformIterator(settings)).andReturn(iter);
    replay(logic);
    
    RunningQuery query = new RunningQuery(connector, connectionPriority, logic, settings, methodAuths, principal, new QueryMetricFactoryImpl());
    
    verify(logic);
    
    // extra tests to verify setConnection worked. Would rather mock and don't really like multiple asserts per test, but there is too much setup
    assertEquals(connector, query.getConnection());
    assertEquals(iter, query.getTransformIterator());
}
 

@SuppressWarnings("unchecked")
@Override
public Iterator<Resource> listChildren() {
    return new TransformIterator(new FilterIterator(super.listChildren(), predicate), new Transformer() {
        public Object transform(Object o) {
            return new PredicatedResourceWrapper((Resource) o, predicate);
        }
    });
}
 

@Override
@SuppressWarnings("unchecked")
public TransformIterator getTransformIterator(Query settings) {
    TransformIterator origIter = super.getTransformIterator(settings);
    QueryLogicTransformer transformer = super.getTransformer(settings);
    DashboardSummary summary = new DashboardSummary(settings.getEndDate());
    while (origIter.hasNext()) {
        EventBase event = (EventBase) transformer.transform(origIter.next());
        DashboardFields.addEvent(summary, event);
    }
    
    return new TransformIterator(Arrays.asList(summary).iterator(), this);
}
 

@Test
public void testConstructor_NoArg() throws Exception {
    // Run the test
    PowerMock.replayAll();
    RunningQuery subject = new RunningQuery();
    Exception result1 = null;
    try {
        subject.next();
    } catch (NullPointerException e) {
        result1 = e;
    }
    Connector result2 = subject.getConnection();
    Priority result3 = subject.getConnectionPriority();
    QueryLogic<?> result4 = subject.getLogic();
    Query result5 = subject.getSettings();
    TransformIterator result6 = subject.getTransformIterator();
    Set<Authorizations> result7 = subject.getCalculatedAuths();
    PowerMock.verifyAll();
    
    // Verify results
    assertNotNull("Expected an exception to be thrown due to uninitialized instance variables", result1);
    
    assertNull("Expected a null connector", result2);
    
    assertNull("Expected a null priority", result3);
    
    assertNull("Expected null logic", result4);
    
    assertNull("Expected a null query (a.k.a. settings)", result5);
    
    assertNull("Expected a null iterator", result6);
    
    assertNull("Expected a null set of authorizations", result7);
}
 

@Test
public void testQueryLogicNoData() throws Exception {
    List<BaseQueryLogic<?>> logics = new ArrayList<>();
    TestQueryLogic logic1 = new TestQueryLogic();
    TestQueryLogic2 logic2 = new TestQueryLogic2();
    logics.add(logic1);
    logics.add(logic2);
    
    QueryImpl settings = new QueryImpl();
    settings.setPagesize(100);
    settings.setQueryAuthorizations(auths.serialize());
    settings.setQuery("FOO == 'BAR'");
    settings.setParameters(new HashSet<>());
    settings.setId(UUID.randomUUID());
    
    CompositeQueryLogic c = new CompositeQueryLogic();
    /**
     * RunningQuery.setupConnection()
     */
    c.setQueryLogics(logics);
    c.initialize((Connector) null, (Query) settings, Collections.singleton(auths));
    c.setupQuery(null);
    TransformIterator iter = c.getTransformIterator((Query) settings);
    
    /**
     * RunningQuery.next() - iterate over results coming from tablet server through the TransformIterator to turn them into the objects.
     */
    List<Object> results = new ArrayList<>();
    while (iter.hasNext()) {
        Object o = iter.next();
        if (null == o)
            break;
        Assert.assertTrue(o instanceof TestQueryResponse);
        results.add((TestQueryResponse) o);
    }
    Assert.assertEquals(0, results.size());
    c.close();
    
}
 

/**
 * Executes test cases that use {@link CountingShardQueryLogic}.
 *
 * @param query
 *            query for evaluation
 * @throws Exception
 *             error condition during execution of query
 */
protected void runCountTest(String query) throws Exception {
    Date[] startEndDate = this.dataManager.getShardStartEndDate();
    if (log.isDebugEnabled()) {
        log.debug("  count query[" + query + "]  start(" + YMD_DateFormat.format(startEndDate[0]) + ")  end(" + YMD_DateFormat.format(startEndDate[1])
                        + ")");
    }
    
    QueryImpl q = new QueryImpl();
    q.setBeginDate(startEndDate[0]);
    q.setEndDate(startEndDate[1]);
    q.setQuery(query);
    
    q.setId(UUID.randomUUID());
    q.setPagesize(Integer.MAX_VALUE);
    q.setQueryAuthorizations(auths.toString());
    
    RunningQuery runner = new RunningQuery(connector, AccumuloConnectionFactory.Priority.NORMAL, this.countLogic, q, "", principal,
                    new QueryMetricFactoryImpl());
    TransformIterator it = runner.getTransformIterator();
    ShardQueryCountTableTransformer ctt = (ShardQueryCountTableTransformer) it.getTransformer();
    EventQueryResponseBase resp = (EventQueryResponseBase) ctt.createResponse(runner.next());
    
    Collection<String> expect = getExpectedKeyResponse(query);
    
    List<EventBase> events = resp.getEvents();
    Assert.assertEquals(1, events.size());
    EventBase<?,?> event = events.get(0);
    List<?> fields = event.getFields();
    Assert.assertEquals(1, fields.size());
    FieldBase<?> count = (FieldBase) fields.get(0);
    String val = count.getValueString();
    if (log.isDebugEnabled()) {
        log.debug("expected count(" + expect.size() + ") actual count(" + val + ")");
    }
    Assert.assertEquals("" + expect.size(), val);
}
 

@Test
public void testUniqueness() {
    Random random = new Random(2000);
    List<Document> input = new ArrayList<>();
    for (int i = 0; i < 100; i++) {
        input.add(createDocument(random, false));
    }
    
    // choose three fields such that the number of unique document is less than half the number of documents but greater than 10
    Set<String> fields = new HashSet<>();
    int expected = input.size();
    while (expected > input.size() / 2 || expected < 10) {
        fields.clear();
        while (fields.size() < 3) {
            fields.add("Attr" + random.nextInt(100));
        }
        expected = countUniqueness(input, fields);
    }
    
    Transformer docToEntry = input1 -> {
        Document d = (Document) input1;
        return Maps.immutableEntry(d.getMetadata(), d);
    };
    TransformIterator inputIterator = new TransformIterator(input.iterator(), docToEntry);
    fields = fields.stream().map(field -> (random.nextBoolean() ? '$' + field : field)).collect(Collectors.toSet());
    UniqueTransform transform = new UniqueTransform(fields);
    Iterator iter = Iterators.transform(inputIterator, transform);
    
    List<Object> eventList = Lists.newArrayList();
    while (iter.hasNext()) {
        Object next = iter.next();
        if (next != null) {
            eventList.add(next);
        }
    }
    
    Assert.assertEquals(expected, eventList.size());
    Assert.assertNull(transform.apply(null));
}
 

protected void doGet(SlingHttpServletRequest request, SlingHttpServletResponse response) throws IOException {
    final String PARAMETER_OFFSET = "_commerce_offset";
    final String PARAMETER_LIMIT = "_commerce_limit";
    final String PARAMETER_COMMERCE_TYPE = "_commerce_commerce_type";
    final Config cfg = new Config(request.getResource().getChild(Config.DATASOURCE));
    final SlingScriptHelper sling = ((SlingBindings) request.getAttribute(SlingBindings.class.getName())).getSling();
    final ExpressionHelper ex = new ExpressionHelper(sling.getService(ExpressionResolver.class), request);
    final String itemRT = cfg.get("itemResourceType", String.class);
    final long offset = ex.get(cfg.get("offset", "0"), long.class);
    final long limit = ex.get(cfg.get("limit", "20"), long.class);
    final String commerceType = ex.get(cfg.get("commerceType", "product"), String.class);

    Map<String, Object> queryParameters = new HashMap<>(request.getParameterMap());
    queryParameters.put(PARAMETER_OFFSET, String.valueOf(offset));
    queryParameters.put(PARAMETER_LIMIT, String.valueOf(limit));
    queryParameters.put(PARAMETER_COMMERCE_TYPE, commerceType);

    final String rootPath = request.getParameter("root");

    String rootCategoryId = new CatalogSearchSupport(request.getResourceResolver()).findCategoryId(rootPath);
    if (rootCategoryId != null) {
        queryParameters.put(CATEGORY_ID_PARAMETER, rootCategoryId);
        queryParameters.put(CATEGORY_PATH_PARAMETER, rootPath);
    }
    String queryString = new ObjectMapper().writeValueAsString(queryParameters);
    try {
        Iterator<Resource> virtualResults = request.getResourceResolver().findResources(queryString, VIRTUAL_PRODUCT_QUERY_LANGUAGE);

        final DataSource ds = new SimpleDataSource(new TransformIterator<>(virtualResults, r -> new ResourceWrapper(r) {
            public String getResourceType() {
                return itemRT;
            }
        }));

        request.setAttribute(DataSource.class.getName(), ds);
    } catch (Exception x) {
        response.sendError(500, x.getMessage());
        LOGGER.error("Error finding resources", x);
    }
}
 

@Test
public void testConstructor_Copy() throws Exception {
    // Set expectations
    expect(this.copy.getMarkingFunctions()).andReturn(null);
    expect(this.copy.getResponseObjectFactory()).andReturn(null);
    expect(this.copy.getLogicName()).andReturn("logicName");
    expect(this.copy.getLogicDescription()).andReturn("logicDescription");
    expect(this.copy.getAuditType(null)).andReturn(Auditor.AuditType.ACTIVE);
    expect(this.copy.getTableName()).andReturn("tableName");
    expect(this.copy.getMaxResults()).andReturn(Long.MAX_VALUE);
    expect(this.copy.getMaxWork()).andReturn(10L);
    expect(this.copy.getMaxPageSize()).andReturn(25);
    expect(this.copy.getPageByteTrigger()).andReturn(1024L);
    expect(this.copy.getCollectQueryMetrics()).andReturn(false);
    expect(this.copy.getConnPoolName()).andReturn("connPool1");
    expect(this.copy.getBaseIteratorPriority()).andReturn(100);
    expect(this.copy.getPrincipal()).andReturn(null);
    RoleManager roleManager = new EasyRoleManager();
    expect(this.copy.getRoleManager()).andReturn(roleManager);
    expect(this.copy.getSelectorExtractor()).andReturn(null);
    expect(this.copy.getBypassAccumulo()).andReturn(false);
    
    // Run the test
    PowerMock.replayAll();
    BaseQueryLogic<Object> subject = new TestQueryLogic<>(this.copy);
    int result1 = subject.getMaxPageSize();
    long result2 = subject.getPageByteTrigger();
    TransformIterator result3 = subject.getTransformIterator(this.query);
    PowerMock.verifyAll();
    
    // Verify results
    assertEquals("Incorrect max page size", 25, result1);
    assertEquals("Incorrect page byte trigger", 1024L, result2);
    assertNotNull("Iterator should not be null", result3);
}
 

public void setTransformIterator(TransformIterator transformIterator) {
    this.transformIterator = transformIterator;
}
 

@Test
public void testQueryLogicNoDataLogic2() throws Exception {
    List<BaseQueryLogic<?>> logics = new ArrayList<>();
    TestQueryLogic logic1 = new TestQueryLogic();
    TestQueryLogic2 logic2 = new TestQueryLogic2();
    logics.add(logic1);
    logics.add(logic2);
    
    logic1.getData().put(key1, value1);
    logic1.getData().put(key2, value2);
    logic1.getData().put(key3, value3);
    logic1.getData().put(key4, value4);
    logic1.getData().put(key5, value5);
    logic1.getData().put(key6, value6);
    logic1.getData().put(key7, value7);
    logic1.getData().put(key8, value8);
    
    QueryImpl settings = new QueryImpl();
    settings.setPagesize(100);
    settings.setQueryAuthorizations(auths.serialize());
    settings.setQuery("FOO == 'BAR'");
    settings.setParameters(new HashSet<>());
    settings.setId(UUID.randomUUID());
    
    CompositeQueryLogic c = new CompositeQueryLogic();
    // max.results.override is set to -1 when it is not passed in as it is an optional paramter
    logic2.setMaxResults(-1);
    logic1.setMaxResults(-1);
    /**
     * RunningQuery.setupConnection()
     */
    c.setQueryLogics(logics);
    c.initialize((Connector) null, (Query) settings, Collections.singleton(auths));
    c.setupQuery(null);
    TransformIterator iter = c.getTransformIterator((Query) settings);
    
    /**
     * RunningQuery.next() - iterate over results coming from tablet server through the TransformIterator to turn them into the objects.
     */
    List<Object> results = new ArrayList<>();
    while (iter.hasNext()) {
        Object o = iter.next();
        if (null == o)
            break;
        Assert.assertTrue(o instanceof TestQueryResponse);
        results.add((TestQueryResponse) o);
    }
    Assert.assertEquals(8, results.size());
    ResultsPage page = new ResultsPage(results, Status.COMPLETE);
    
    /**
     * QueryExecutorBean.next() - transform list of objects into JAXB response
     */
    TestQueryResponseList response = (TestQueryResponseList) c.getTransformer((Query) settings).createResponse(page);
    Assert.assertEquals(8, response.getResponses().size());
    for (TestQueryResponse r : response.getResponses()) {
        Assert.assertNotNull(r);
    }
    
    c.close();
    
}
 

@Test
public void testQueryLogicNoDataLogic1() throws Exception {
    List<BaseQueryLogic<?>> logics = new ArrayList<>();
    TestQueryLogic logic1 = new TestQueryLogic();
    TestQueryLogic2 logic2 = new TestQueryLogic2();
    logics.add(logic1);
    logics.add(logic2);
    
    logic2.getData().put(key1, value1);
    logic2.getData().put(key2, value2);
    logic2.getData().put(key3, value3);
    logic2.getData().put(key4, value4);
    logic2.getData().put(key5, value5);
    logic2.getData().put(key6, value6);
    logic2.getData().put(key7, value7);
    logic2.getData().put(key8, value8);
    
    QueryImpl settings = new QueryImpl();
    settings.setPagesize(100);
    settings.setQueryAuthorizations(auths.serialize());
    settings.setQuery("FOO == 'BAR'");
    settings.setParameters(new HashSet<>());
    settings.setId(UUID.randomUUID());
    
    CompositeQueryLogic c = new CompositeQueryLogic();
    // max.results.override is set to -1 when it is not passed in as it is an optional paramter
    logic2.setMaxResults(-1);
    logic1.setMaxResults(-1);
    /**
     * RunningQuery.setupConnection()
     */
    c.setQueryLogics(logics);
    c.initialize((Connector) null, (Query) settings, Collections.singleton(auths));
    c.setupQuery(null);
    TransformIterator iter = c.getTransformIterator((Query) settings);
    
    /**
     * RunningQuery.next() - iterate over results coming from tablet server through the TransformIterator to turn them into the objects.
     */
    List<Object> results = new ArrayList<>();
    while (iter.hasNext()) {
        Object o = iter.next();
        if (null == o)
            break;
        Assert.assertTrue(o instanceof TestQueryResponse);
        results.add((TestQueryResponse) o);
    }
    Assert.assertEquals(8, results.size());
    ResultsPage page = new ResultsPage(results, Status.COMPLETE);
    
    /**
     * QueryExecutorBean.next() - transform list of objects into JAXB response
     */
    TestQueryResponseList response = (TestQueryResponseList) c.getTransformer((Query) settings).createResponse(page);
    Assert.assertEquals(8, response.getResponses().size());
    for (TestQueryResponse r : response.getResponses()) {
        Assert.assertNotNull(r);
    }
    
    c.close();
    
}
 

@Test
// testQueryLogic with max.results.override is set
public void testQueryLogicWithMaxResultsOverride() throws Exception {
    Logger.getLogger(CompositeQueryLogic.class).setLevel(Level.TRACE);
    Logger.getLogger(CompositeQueryLogicResults.class).setLevel(Level.TRACE);
    Logger.getLogger(CompositeQueryLogicTransformer.class).setLevel(Level.TRACE);
    List<BaseQueryLogic<?>> logics = new ArrayList<>();
    TestQueryLogic logic1 = new TestQueryLogic();
    TestQueryLogic2 logic2 = new TestQueryLogic2();
    logics.add(logic1);
    logics.add(logic2);
    
    logic1.getData().put(key1, value1);
    logic1.getData().put(key2, value2);
    logic2.getData().put(key3, value3);
    logic2.getData().put(key4, value4);
    logic1.getData().put(key5, value5);
    logic1.getData().put(key6, value6);
    logic2.getData().put(key7, value7);
    logic2.getData().put(key8, value8);
    
    QueryImpl settings = new QueryImpl();
    settings.setPagesize(100);
    settings.setQueryAuthorizations(auths.toString());
    settings.setQuery("FOO == 'BAR'");
    settings.setParameters(new HashSet<>());
    settings.setId(UUID.randomUUID());
    
    CompositeQueryLogic c = new CompositeQueryLogic();
    // max.results.override is set to -1 when it is not passed in as it is an optional parameter
    logic1.setMaxResults(0);
    logic2.setMaxResults(4);
    /**
     * RunningQuery.setupConnection()
     */
    c.setQueryLogics(logics);
    c.initialize((Connector) null, (Query) settings, Collections.singleton(auths));
    c.setupQuery(null);
    TransformIterator iter = c.getTransformIterator((Query) settings);
    
    /**
     * RunningQuery.next() - iterate over results coming from tablet server through the TransformIterator to turn them into the objects.
     */
    List<Object> results = new ArrayList<>();
    while (iter.hasNext()) {
        Object o = iter.next();
        if (null == o)
            break;
        Assert.assertTrue(o instanceof TestQueryResponse);
        results.add((TestQueryResponse) o);
    }
    Assert.assertEquals(4, results.size());
    ResultsPage page = new ResultsPage(results, Status.COMPLETE);
    
    /**
     * QueryExecutorBean.next() - transform list of objects into JAXB response
     */
    TestQueryResponseList response = (TestQueryResponseList) c.getTransformer((Query) settings).createResponse(page);
    Assert.assertEquals(4, response.getResponses().size());
    for (TestQueryResponse r : response.getResponses()) {
        Assert.assertNotNull(r);
    }
    
    c.close();
    
}
 

@Test
// testQueryLogic with max.results.override not set
public void testQueryLogic() throws Exception {
    Logger.getLogger(CompositeQueryLogic.class).setLevel(Level.TRACE);
    Logger.getLogger(CompositeQueryLogicResults.class).setLevel(Level.TRACE);
    Logger.getLogger(CompositeQueryLogicTransformer.class).setLevel(Level.TRACE);
    List<BaseQueryLogic<?>> logics = new ArrayList<>();
    TestQueryLogic logic1 = new TestQueryLogic();
    TestQueryLogic2 logic2 = new TestQueryLogic2();
    logics.add(logic1);
    logics.add(logic2);
    
    logic1.getData().put(key1, value1);
    logic1.getData().put(key2, value2);
    logic2.getData().put(key3, value3);
    logic2.getData().put(key4, value4);
    logic1.getData().put(key5, value5);
    logic1.getData().put(key6, value6);
    logic2.getData().put(key7, value7);
    logic2.getData().put(key8, value8);
    
    QueryImpl settings = new QueryImpl();
    settings.setPagesize(100);
    settings.setQueryAuthorizations(auths.toString());
    settings.setQuery("FOO == 'BAR'");
    settings.setParameters(new HashSet<>());
    settings.setId(UUID.randomUUID());
    
    CompositeQueryLogic c = new CompositeQueryLogic();
    // max.results.override is set to -1 when it is not passed in as it is an optional paramter
    logic1.setMaxResults(-1);
    logic2.setMaxResults(-1);
    /**
     * RunningQuery.setupConnection()
     */
    c.setQueryLogics(logics);
    c.initialize((Connector) null, (Query) settings, Collections.singleton(auths));
    c.setupQuery(null);
    TransformIterator iter = c.getTransformIterator((Query) settings);
    
    /**
     * RunningQuery.next() - iterate over results coming from tablet server through the TransformIterator to turn them into the objects.
     */
    List<Object> results = new ArrayList<>();
    while (iter.hasNext()) {
        Object o = iter.next();
        if (null == o)
            break;
        Assert.assertTrue(o instanceof TestQueryResponse);
        results.add((TestQueryResponse) o);
    }
    Assert.assertEquals(8, results.size());
    ResultsPage page = new ResultsPage(results, Status.COMPLETE);
    
    /**
     * QueryExecutorBean.next() - transform list of objects into JAXB response
     */
    TestQueryResponseList response = (TestQueryResponseList) c.getTransformer((Query) settings).createResponse(page);
    Assert.assertEquals(8, response.getResponses().size());
    for (TestQueryResponse r : response.getResponses()) {
        Assert.assertNotNull(r);
    }
    
    c.close();
    
}
 

public TransformIterator getTransformIterator() {
    return iter;
}
 

@Override
public TransformIterator getTransformIterator(Query settings) {
    return new DatawaveTransformIterator(this.iterator(), this.getTransformer(settings));
}
 

@Override
public TransformIterator getTransformIterator(Query settings) {
    // The objects put into the pageQueue have already been transformed.
    // We will iterate over the pagequeue with the No-Op transformer
    return new TransformIterator(results.iterator(), NOPTransformer.nopTransformer());
}
 

protected BaseQueryResponse runTestQueryWithGrouping(Map<String,Integer> expected, String querystr, Date startDate, Date endDate,
                Map<String,String> extraParms, Connector connector) throws Exception {
    log.debug("runTestQueryWithGrouping");
    
    QueryImpl settings = new QueryImpl();
    settings.setBeginDate(startDate);
    settings.setEndDate(endDate);
    settings.setPagesize(Integer.MAX_VALUE);
    settings.setQueryAuthorizations(auths.serialize());
    settings.setQuery(querystr);
    settings.setParameters(extraParms);
    settings.setId(UUID.randomUUID());
    
    log.debug("query: " + settings.getQuery());
    log.debug("logic: " + settings.getQueryLogicName());
    
    GenericQueryConfiguration config = logic.initialize(connector, settings, authSet);
    logic.setupQuery(config);
    
    DocumentTransformer transformer = (DocumentTransformer) (logic.getTransformer(settings));
    TransformIterator iter = new DatawaveTransformIterator(logic.iterator(), transformer);
    List<Object> eventList = new ArrayList<>();
    while (iter.hasNext()) {
        eventList.add(iter.next());
    }
    
    BaseQueryResponse response = transformer.createResponse(eventList);
    
    // un-comment to look at the json output
    ObjectMapper mapper = new ObjectMapper();
    mapper.enable(MapperFeature.USE_WRAPPER_NAME_AS_PROPERTY_NAME);
    mapper.writeValue(new File("/tmp/grouped2.json"), response);
    
    Assert.assertTrue(response instanceof DefaultEventQueryResponse);
    DefaultEventQueryResponse eventQueryResponse = (DefaultEventQueryResponse) response;
    
    Assert.assertEquals("Got the wrong number of events", expected.size(), (long) eventQueryResponse.getReturnedEvents());
    
    for (EventBase event : eventQueryResponse.getEvents()) {
        
        String genderKey = "";
        String ageKey = "";
        Integer value = null;
        for (Object field : event.getFields()) {
            FieldBase fieldBase = (FieldBase) field;
            switch (fieldBase.getName()) {
                case "COUNT":
                    value = Integer.valueOf(fieldBase.getValueString());
                    break;
                case "GENDER":
                    genderKey = fieldBase.getValueString();
                    break;
                case "GEN":
                    genderKey = fieldBase.getValueString();
                    break;
                case "AGE":
                    ageKey = fieldBase.getValueString();
                    break;
                case "AG":
                    ageKey = fieldBase.getValueString();
                    break;
            }
        }
        
        log.debug("mapping is " + genderKey + "-" + ageKey + " count:" + value);
        String key;
        if (!genderKey.isEmpty() && !ageKey.isEmpty()) {
            key = genderKey + "-" + ageKey;
        } else if (!genderKey.isEmpty()) {
            key = genderKey;
        } else {
            key = ageKey;
        }
        Assert.assertEquals(expected.get(key), value);
    }
    return response;
}
 

protected BaseQueryResponse runTestQueryWithGrouping(Map<String,Integer> expected, String querystr, Date startDate, Date endDate,
                Map<String,String> extraParms, Connector connector) throws Exception {
    log.debug("runTestQueryWithGrouping");
    
    QueryImpl settings = new QueryImpl();
    settings.setBeginDate(startDate);
    settings.setEndDate(endDate);
    settings.setPagesize(Integer.MAX_VALUE);
    settings.setQueryAuthorizations(auths.serialize());
    settings.setQuery(querystr);
    settings.setParameters(extraParms);
    settings.setId(UUID.randomUUID());
    
    log.debug("query: " + settings.getQuery());
    log.debug("logic: " + settings.getQueryLogicName());
    
    GenericQueryConfiguration config = logic.initialize(connector, settings, authSet);
    logic.setupQuery(config);
    
    DocumentTransformer transformer = (DocumentTransformer) (logic.getTransformer(settings));
    TransformIterator iter = new DatawaveTransformIterator(logic.iterator(), transformer);
    List<Object> eventList = new ArrayList<>();
    while (iter.hasNext()) {
        eventList.add(iter.next());
    }
    
    BaseQueryResponse response = transformer.createResponse(eventList);
    
    // un-comment to look at the json output
    ObjectMapper mapper = new ObjectMapper();
    mapper.enable(MapperFeature.USE_WRAPPER_NAME_AS_PROPERTY_NAME);
    mapper.writeValue(new File("/tmp/grouped2.json"), response);
    
    Assert.assertTrue(response instanceof DefaultEventQueryResponse);
    DefaultEventQueryResponse eventQueryResponse = (DefaultEventQueryResponse) response;
    
    Assert.assertEquals("Got the wrong number of events", expected.size(), (long) eventQueryResponse.getReturnedEvents());
    
    for (EventBase event : eventQueryResponse.getEvents()) {
        
        String genderKey = "";
        String ageKey = "";
        Integer value = null;
        for (Object field : event.getFields()) {
            FieldBase fieldBase = (FieldBase) field;
            switch (fieldBase.getName()) {
                case "COUNT":
                    value = Integer.valueOf(fieldBase.getValueString());
                    break;
                case "GENDER":
                    genderKey = fieldBase.getValueString();
                    break;
                case "GEN":
                    genderKey = fieldBase.getValueString();
                    break;
                case "AGE":
                    ageKey = fieldBase.getValueString();
                    break;
                case "AG":
                    ageKey = fieldBase.getValueString();
                    break;
            }
        }
        
        log.debug("mapping is " + genderKey + "-" + ageKey + " count:" + value);
        String key;
        if (!genderKey.isEmpty() && !ageKey.isEmpty()) {
            key = genderKey + "-" + ageKey;
        } else if (!genderKey.isEmpty()) {
            key = genderKey;
        } else {
            key = ageKey;
        }
        Assert.assertEquals(expected.get(key), value);
    }
    return response;
}
 

protected void runTestQueryWithUniqueness(Set<Set<String>> expected, String querystr, Date startDate, Date endDate, Map<String,String> extraParms,
                Connector connector) throws Exception {
    log.debug("runTestQueryWithUniqueness");
    
    QueryImpl settings = new QueryImpl();
    settings.setBeginDate(startDate);
    settings.setEndDate(endDate);
    settings.setPagesize(Integer.MAX_VALUE);
    settings.setQueryAuthorizations(auths.serialize());
    settings.setQuery(querystr);
    settings.setParameters(extraParms);
    settings.setId(UUID.randomUUID());
    
    log.debug("query: " + settings.getQuery());
    log.debug("logic: " + settings.getQueryLogicName());
    
    GenericQueryConfiguration config = logic.initialize(connector, settings, authSet);
    logic.setupQuery(config);
    
    DocumentTransformer transformer = (DocumentTransformer) (logic.getTransformer(settings));
    TransformIterator iter = new DatawaveTransformIterator(logic.iterator(), transformer);
    List<Object> eventList = new ArrayList<>();
    while (iter.hasNext()) {
        eventList.add(iter.next());
    }
    
    BaseQueryResponse response = transformer.createResponse(eventList);
    
    // un-comment to look at the json output
    // ObjectMapper mapper = new ObjectMapper();
    // mapper.enable(MapperFeature.USE_WRAPPER_NAME_AS_PROPERTY_NAME);
    // mapper.writeValue(new File("/tmp/grouped2.json"), response);
    
    Assert.assertTrue(response instanceof DefaultEventQueryResponse);
    DefaultEventQueryResponse eventQueryResponse = (DefaultEventQueryResponse) response;
    
    for (EventBase event : eventQueryResponse.getEvents()) {
        boolean found = false;
        for (Iterator<Set<String>> it = expected.iterator(); it.hasNext();) {
            Set<String> expectedSet = it.next();
            if (expectedSet.contains(event.getMetadata().getInternalId())) {
                it.remove();
                found = true;
                break;
            }
        }
        Assert.assertTrue(found);
    }
    Assert.assertTrue(expected.isEmpty());
}
 

@Test
public void runQuery() throws Exception {
    SimpleDateFormat format = new SimpleDateFormat("yyyyMMdd");
    QueryImpl q = new QueryImpl();
    q.setBeginDate(format.parse("20180101"));
    q.setEndDate(format.parse("20200101"));
    q.setQuery("ID == 'id-001'");
    q.setQueryAuthorizations(auths.toString());
    q.setPagesize(Integer.MAX_VALUE);
    q.setId(UUID.randomUUID());
    
    /*
     * The reduced response parameter causes components of the attribute metadata, including timestamp, not to be written or read when serializing and
     * deserializing
     */
    q.addParameter("reduced.response", "true");
    q.setQueryLogicName("EventQuery");
    
    RunningQuery query = new RunningQuery(connector, AccumuloConnectionFactory.Priority.NORMAL, logic, q, "", datawavePrincipal,
                    new QueryMetricFactoryImpl());
    TransformIterator<?,?> it = query.getTransformIterator();
    AbstractQueryLogicTransformer<?,?> et = (DocumentTransformer) it.getTransformer();
    
    CardinalityConfiguration cconf = new CardinalityConfiguration();
    Set<String> cardFields = new HashSet<String>();
    cardFields.add("ID");
    cconf.setCardinalityFields(cardFields);
    
    Map<String,String> cmap = new HashMap<String,String>();
    cmap.put("ID", "ID");
    cconf.setCardinalityFieldReverseMapping(cmap);
    
    String path = temporaryFolder.toAbsolutePath().toString();
    
    cconf.setOutputFileDirectory(path);
    cconf.setCardinalityUidField("ID");
    cconf.setFlushThreshold(1);
    
    ((DocumentTransformer) it.getTransformer()).setCardinalityConfiguration(cconf);
    EventQueryResponseBase response = (EventQueryResponseBase) et.createResponse(query.next());
    
    // Wait for cardinality info to be written to the file
    Thread.currentThread().sleep(1000);
    
    Assert.assertTrue(response.getReturnedEvents().intValue() > 0);
    
    boolean createdDocFile = false;
    boolean readDocFile = false;
    File folder = new File(path);
    for (File f : folder.listFiles()) {
        if (f.getName().contains("document")) {
            createdDocFile = true;
            CardinalityRecord rcc = CardinalityRecord.readFromDisk(f);
            if (null != rcc) {
                for (DateFieldValueCardinalityRecord dr : rcc.getCardinalityMap().values()) {
                    Assert.assertEquals("20190101", dr.getEventDate());
                    readDocFile = true;
                }
            }
        }
    }
    Assert.assertTrue(createdDocFile);
    Assert.assertTrue(readDocFile);
    
}
 

@Override
public TransformIterator getTransformIterator(Query settings) {
    return new MyCountAggregatingIterator(this.iterator(), getTransformer(settings), this.maxUniqueValues);
}
 

@Override
public TransformIterator getTransformIterator(Query settings) {
    return new CountAggregatingIterator(this.iterator(), getTransformer(settings));
}
 

protected void doGet(SlingHttpServletRequest request, SlingHttpServletResponse response){

        final SlingScriptHelper sling = getScriptHelper(request);

        final ExpressionHelper ex = new ExpressionHelper(sling.getService(ExpressionResolver.class), request);
        final Config dsCfg = new Config(request.getResource().getChild(Config.DATASOURCE));
        final CommerceBasePathsService cbps = sling.getService(CommerceBasePathsService.class);

        final String query = ex.getString(dsCfg.get("query", String.class));

        final String parentPath;
        final String searchName;
        final String rootPath = ex.getString(dsCfg.get("rootPath", cbps.getProductsBasePath()));

        if (query != null) {
            final int slashIndex = query.lastIndexOf('/');
            if (slashIndex < 0) {
                parentPath = rootPath;
                searchName = query.toLowerCase();
            } else if (!query.startsWith(rootPath)) {
                parentPath = rootPath;
                searchName = null;
            } else if (slashIndex == query.length() - 1) {
                parentPath = query;
                searchName = null;
            } else {
                parentPath = query.substring(0, slashIndex + 1);
                searchName = query.substring(slashIndex + 1).toLowerCase();
            }
        } else {
            parentPath = ex.getString(dsCfg.get("path", String.class));
            searchName = null;
        }

        final Resource parent = request.getResourceResolver().getResource(parentPath);

        final DataSource ds;
        if (parent == null) {
            ds = EmptyDataSource.instance();
        } else {
            final Integer offset = ex.get(dsCfg.get("offset", String.class), Integer.class);
            final Integer limit = ex.get(dsCfg.get("limit", String.class), Integer.class);
            final String itemRT = dsCfg.get("itemResourceType", String.class);
            final String filter = ex.getString(dsCfg.get("filter", String.class));

            final Collection<Predicate<Resource>> predicates = new ArrayList<>(2);
            predicates.add(createPredicate(filter));

            if (searchName != null) {
                final Pattern searchNamePattern = Pattern.compile(Pattern.quote(searchName), Pattern.CASE_INSENSITIVE);
                predicates.add(resource -> searchNamePattern.matcher(resource.getName()).lookingAt());
            }

            final Predicate predicate = PredicateUtils.allPredicate(predicates);
            final Transformer transformer = createTransformer(itemRT, predicate);


            final List<Resource> list;
            if (FILTER_CATEGORY.equals(filter)) {
                class CategoryFinder extends AbstractResourceVisitor {
                    private CategoryPredicate categoryPredicate = new CategoryPredicate();
                    private List<Resource> categories = new ArrayList<Resource>();
                    @Override
                    protected void visit(Resource res) {
                        if (categoryPredicate.evaluate(res)) {
                            categories.add(res);
                        }
                    }
                };
                CategoryFinder categoryFinder = new CategoryFinder();
                categoryFinder.accept(parent);
                list = IteratorUtils.toList(new FilterIterator(categoryFinder.categories.iterator(), predicate));
            } else {
                list =IteratorUtils.toList(new FilterIterator(parent.listChildren(), predicate));
            }

            //force reloading the children of the root node to hit the virtual resource provider
            if (rootPath.equals(parentPath)) {
                for (int i = 0; i < list.size(); i++) {
                    list.set(i, request.getResourceResolver().getResource(list.get(i).getPath()));
                }
            }

            @SuppressWarnings("unchecked")
            DataSource datasource = new AbstractDataSource() {

                public Iterator<Resource> iterator() {
                    Collections.sort(list, Comparator.comparing(Resource::getName));
                    return new TransformIterator(new PagingIterator<>(list.iterator(), offset, limit), transformer);
                }
            };

            ds = datasource;
        }

        request.setAttribute(DataSource.class.getName(), ds);
    }
 

/**
 * Allows for the customization of handling query results, e.g. allows for aggregation of query results before returning to the client.
 * 
 * @param settings
 *            The query settings object
 * @return Return a TransformIterator for the QueryLogic implementation
 */
TransformIterator getTransformIterator(Query settings);