Java Code Examples for com.googlecode.cqengine.IndexedCollection#addIndex()

@Test
public void testGetFrequencyDistribution() {
    IndexedCollection<Car> cars = createIndexedCollectionOfCars(20);

    // Add an unsorted index on Car.MANUFACTURER (a HashIndex)...
    cars.addIndex(HashIndex.onAttribute(Car.MANUFACTURER));
    MetadataEngine<Car> metadataEngine = cars.getMetadataEngine();

    // Access metadata for Car.MANUFACTURER attribute.
    // Because we call getAttributeMetadata() (and not getSortedAttributeMetadata()),
    // values will be returned in no particular order...
    AttributeMetadata<String, Car> sortedAttributeMetadata = metadataEngine.getAttributeMetadata(Car.MANUFACTURER);

    // Call AttributeMetadata.getFrequencyDistribution() to retrieve distinct keys and counts in no particular order.
    // We retrieve into a Set (not a List), to assert the expected values were returned, without asserting any order...
    assertEquals(
            asSet(frequency("Ford", 6), frequency("BMW", 2), frequency("Toyota", 6), frequency("Honda", 6)),
            sortedAttributeMetadata.getFrequencyDistribution().collect(toSet())
    );
}
 

@Test
public void testGetKeysAndValues() {
    IndexedCollection<Car> cars = new ConcurrentIndexedCollection<>();
    Car car1 = new Car(1, "Ford", "Taurus", Car.Color.GREEN, 4, 1000.0, emptyList(), Collections.emptyList());
    Car car2 = new Car(2, "Honda", "Civic", Car.Color.BLUE, 4, 2000.0, emptyList(), Collections.emptyList());
    Car car3 = new Car(3, "Honda", "Accord", Car.Color.RED, 4, 3000.0, emptyList(), Collections.emptyList());
    cars.addAll(asList(car1, car2, car3));

    // Add an unsorted index on Car.MANUFACTURER (a HashIndex)...
    cars.addIndex(HashIndex.onAttribute(Car.MANUFACTURER));
    MetadataEngine<Car> metadataEngine = cars.getMetadataEngine();

    // Access metadata for Car.MODEL attribute.
    // Because we call getAttributeMetadata() (and not getSortedAttributeMetadata()),
    // values will be returned in no particular order...
    AttributeMetadata<String, Car> attributeMetadata = metadataEngine.getAttributeMetadata(Car.MANUFACTURER);

    // Call AttributeMetadata.getDistinctKeys() to retrieve distinct keys in no particular order.
    // We retrieve into a Set (not a List), to assert the expected values were returned, without asserting any order...
    assertEquals(
            asSet(keyValue("Ford", car1), keyValue("Honda", car2), keyValue("Honda", car3)),
            attributeMetadata.getKeysAndValues().collect(toSet())
    );
}
 

@Test
public void testExists() {
    // Create an indexed collection (note: could alternatively use CQEngine.copyFrom() existing collection)...
    IndexedCollection<Car> cars = new ConcurrentIndexedCollection<Car>();

    Attribute<Car, String> NAME = new SimpleNullableAttribute<Car, String>("name") {
        public String getValue(Car car, QueryOptions queryOptions) { return car.name; }
    };
    // Add some indexes...
    cars.addIndex(StandingQueryIndex.onQuery(has(NAME)));
    cars.addIndex(StandingQueryIndex.onQuery(not(has(NAME))));

    // Add some objects to the collection...
    cars.add(new Car(1, "ford focus", "great condition, low mileage", Arrays.asList("spare tyre", "sunroof")));
    cars.add(new Car(2, null, "dirty and unreliable, flat tyre", Arrays.asList("spare tyre", "radio")));
    cars.add(new Car(3, "honda civic", "has a flat tyre and high mileage", Arrays.asList("radio")));

    Assert.assertEquals(cars.retrieve(has(NAME)).size(), 2);
    Assert.assertEquals(cars.retrieve(not(has(NAME))).size(), 1);
}
 

@Test
public void testGetDistinctKeysAndCounts() {
    IndexedCollection<Car> collection = new ConcurrentIndexedCollection<Car>();
    SortedKeyStatisticsIndex<String, Car> MODEL_INDEX = NavigableIndex.onAttribute(Car.MODEL);
    collection.addIndex(MODEL_INDEX);

    collection.addAll(CarFactory.createCollectionOfCars(20));

    Set<String> distinctModels = setOf(MODEL_INDEX.getDistinctKeys(noQueryOptions()));
    assertEquals(asList("Accord", "Avensis", "Civic", "Focus", "Fusion", "Hilux", "Insight", "M6", "Prius", "Taurus"), new ArrayList<String>(distinctModels));
    for (String model : distinctModels) {
        assertEquals(Integer.valueOf(2), MODEL_INDEX.getCountForKey(model, noQueryOptions()));
    }

    Set<String> distinctModelsDescending = setOf(MODEL_INDEX.getDistinctKeysDescending(noQueryOptions()));
    assertEquals(asList("Taurus", "Prius", "M6", "Insight", "Hilux", "Fusion", "Focus", "Civic", "Avensis", "Accord"), new ArrayList<String>(distinctModelsDescending));
}
 

@Test
public void testInOne() {
    // Create an indexed collection (note: could alternatively use CQEngine.copyFrom() existing collection)...
    IndexedCollection<Car> cars = new ConcurrentIndexedCollection<Car>();

    Attribute<Car, String> NAME = new SimpleNullableAttribute<Car, String>("name") {
        public String getValue(Car car, QueryOptions queryOptions) {
            return car.name;
        }
    };
    cars.addIndex(NavigableIndex.onAttribute(NAME));

    // Add some objects to the collection...
    cars.add(new Car(1, "ford", null, null));
    cars.add(new Car(2, "honda", null, null));
    cars.add(new Car(3, "toyota", null, null));

    Assert.assertEquals(cars.retrieve(in(NAME, "ford")).size(), 1);
    Assert.assertEquals(cars.retrieve(in(NAME, Collections.singletonList("ford"))).size(), 1);
}
 

@Test
public void testInMany() {
    // Create an indexed collection (note: could alternatively use CQEngine.copyFrom() existing collection)...
    IndexedCollection<Car> cars = new ConcurrentIndexedCollection<Car>();

    Attribute<Car, String> NAME = new SimpleNullableAttribute<Car, String>("name") {
        public String getValue(Car car, QueryOptions queryOptions) {
            return car.name;
        }
    };
    cars.addIndex(NavigableIndex.onAttribute(NAME));

    // Add some objects to the collection...
    cars.add(new Car(1, "ford", null, null));
    cars.add(new Car(2, "honda", null, null));
    cars.add(new Car(3, "toyota", null, null));

    Assert.assertEquals(cars.retrieve(in(NAME, "ford", "honda")).size(), 2);
    Assert.assertEquals(cars.retrieve(in(NAME, Arrays.asList("ford", "honda"))).size(), 2);
}
 

@Test
public void testIndexQuantization_SpanningTwoBucketsMidRange() {
    IndexedCollection<Car> collection = new ConcurrentIndexedCollection<Car>();
    collection.addIndex(NavigableIndex.withQuantizerOnAttribute(IntegerQuantizer.withCompressionFactor(10), Car.CAR_ID));
    collection.addAll(CarFactory.createCollectionOfCars(100));

    // Merge cost should be 20 because this query spans 2 buckets (each containing 10 objects)...
    assertEquals(20, collection.retrieve(between(Car.CAR_ID, 47, 53)).getMergeCost());

    // 7 objects match the query (between is inclusive)...
    assertEquals(7, collection.retrieve(between(Car.CAR_ID, 47, 53)).size());

    // The matching objects are...
    List<Integer> carIdsFound = retrieveCarIds(collection, between(Car.CAR_ID, 47, 53));
    assertEquals(asList(47, 48, 49, 50, 51, 52, 53), carIdsFound);
}
 

public static void main(String[] args) {
    final int NUM_ITERATIONS = 1000;
    final int[] numObjects = {10000, 10000, 100000};
    final double[] selectivityThreshold = {0.0, 0.5, 1.0};

    IndexedCollection<Car> cars = new ConcurrentIndexedCollection<Car>();
    cars.addAll(CarFactory.createCollectionOfCars(1000000));

    cars.addIndex(NavigableIndex.onAttribute(Car.CAR_ID));
    cars.addIndex(NavigableIndex.onAttribute(Car.COLOR));

    for (int n : numObjects) {
        for (double s : selectivityThreshold) {
            long start = System.currentTimeMillis();
            long count = 0;
            for (int i = 0; i < NUM_ITERATIONS; i++) {
                count = countRetrievedResults(cars, n, s);
            }
            long timeTaken = System.currentTimeMillis() - start;
            System.out.println("Number: " + n + ", selectivity threshold: " + s + ", time taken per iteration: " + (timeTaken / (double)NUM_ITERATIONS) + " (count=" + count + ")");
        }
    }
}
 

@Test
public void testDeduplication_Logical() {
    IndexedCollection<Car> cars = new ConcurrentIndexedCollection<Car>();
    cars.add(new Car(1, "Ford", "Focus", BLUE, 5, 1000.0, Collections.<String>emptyList(), Collections.emptyList()));
    cars.addIndex(HashIndex.onAttribute(Car.COLOR));
    cars.addIndex(HashIndex.onAttribute(Car.MANUFACTURER));

    Query<Car> query = or(
            equal(COLOR, BLUE),
            equal(MANUFACTURER, "Ford")
    );
    ResultSet<Car> results;
    results = cars.retrieve(query);
    assertEquals(2, results.size());

    DeduplicationOption deduplicate = deduplicate(DeduplicationStrategy.LOGICAL_ELIMINATION);
    results = cars.retrieve(query, queryOptions(deduplicate));
    assertEquals(1, results.size());
}
 

@Test
public void testAttemptToAccessMetadataWithIndexOnDifferentAttribute() {
    IndexedCollection<Car> cars = createIndexedCollectionOfCars(5);
    // Add an index on a different attribute...
    cars.addIndex(HashIndex.onAttribute(Car.MODEL));
    // Create a mock (different) attribute we will query...
    @SuppressWarnings("unchecked") Attribute<Car, Integer> ATTRIBUTE = Mockito.mock(SimpleAttribute.class);
    Mockito.when(ATTRIBUTE.toString()).thenReturn("ATTRIBUTE");
    IllegalStateException expected = null;
    try {
        cars.getMetadataEngine().getAttributeMetadata(ATTRIBUTE);
    }
    catch (IllegalStateException e) {
        expected = e;
    }
    assertNotNull(expected);
    assertEquals("A KeyStatisticsAttributeIndex has not been added to the collection, and must be added first, to enable metadata to be examined for attribute: ATTRIBUTE", expected.getMessage());
}
 

public static void main(String[] args) {
    // Create an indexed collection (note: could alternatively use CQEngine.copyFrom() existing collection)...
    IndexedCollection<Car> cars = new ConcurrentIndexedCollection<Car>();

    // Add some indexes...
    cars.addIndex(NavigableIndex.onAttribute(Car.CAR_ID));
    cars.addIndex(ReversedRadixTreeIndex.onAttribute(Car.NAME));
    cars.addIndex(SuffixTreeIndex.onAttribute(Car.DESCRIPTION));
    cars.addIndex(HashIndex.onAttribute(Car.FEATURES));

    // Add some objects to the collection...
    cars.add(new Car(1, "ford focus", "great condition, low mileage", Arrays.asList("spare tyre", "sunroof")));
    cars.add(new Car(2, "ford taurus", "dirty and unreliable, flat tyre", Arrays.asList("spare tyre", "radio")));
    cars.add(new Car(3, "honda civic", "has a flat tyre and high mileage", Arrays.asList("radio")));

    // -------------------------- Run some queries --------------------------
    System.out.println("Cars whose name ends with 'vic' or whose id is less than 2:");
    Query<Car> query1 = or(endsWith(Car.NAME, "vic"), lessThan(Car.CAR_ID, 2));
    cars.retrieve(query1).forEach(System.out::println);

    System.out.println("\nCars whose flat tyre can be replaced:");
    Query<Car> query2 = and(contains(Car.DESCRIPTION, "flat tyre"), equal(Car.FEATURES, "spare tyre"));
    cars.retrieve(query2).forEach(System.out::println);


    System.out.println("\nCars which have a sunroof or a radio but are not dirty:");
    Query<Car> query3 = and(in(Car.FEATURES, "sunroof", "radio"), not(contains(Car.DESCRIPTION, "dirty")));
    cars.retrieve(query3).forEach(System.out::println);
}
 

@Test
public void testIndexQuantization_GreaterThan() {
    IndexedCollection<Car> collection = new ConcurrentIndexedCollection<Car>();
    collection.addIndex(NavigableIndex.withQuantizerOnAttribute(IntegerQuantizer.withCompressionFactor(10), Car.CAR_ID));
    collection.addAll(CarFactory.createCollectionOfCars(100));

    assertEquals(4, collection.retrieve(greaterThan(Car.CAR_ID, 95)).size());
    assertEquals(14, collection.retrieve(greaterThan(Car.CAR_ID, 85)).size());
    assertEquals(5, collection.retrieve(greaterThanOrEqualTo(Car.CAR_ID, 95)).size());
    assertEquals(15, collection.retrieve(greaterThanOrEqualTo(Car.CAR_ID, 85)).size());
}
 

@Test
public void testGetDistinctKeys() {
    IndexedCollection<Car> cars = createIndexedCollectionOfCars(20); // the 20 cars will contain 10 distinct models

    // Add a sorted index on Car.MODEL (a NavigableIndex)...
    cars.addIndex(NavigableIndex.onAttribute(Car.MODEL));
    MetadataEngine<Car> metadataEngine = cars.getMetadataEngine();

    // Access metadata for Car.MODEL attribute.
    // Because we call getSortedAttributeMetadata(), values will be returned in sorted order...
    SortedAttributeMetadata<String, Car> sortedAttributeMetadata = metadataEngine.getSortedAttributeMetadata(Car.MODEL);

    // Call SortedAttributeMetadata.getDistinctKeys() to retrieve distinct keys in ascending order.
    // We retrieve into a List in order to assert the ordering of values returned...
    assertEquals(
            asList("Accord", "Avensis", "Civic", "Focus", "Fusion", "Hilux", "Insight", "M6", "Prius", "Taurus"),
            sortedAttributeMetadata.getDistinctKeys().collect(toList())
    );

    // Test specifying range explicitly...
    assertEquals(
            asList("Civic", "Focus", "Fusion", "Hilux", "Insight"),
            sortedAttributeMetadata.getDistinctKeys("Civic", true, "Insight", true).collect(toList())
    );
    assertEquals(
            asList("Focus", "Fusion", "Hilux"),
            sortedAttributeMetadata.getDistinctKeys("Civic", false, "Insight", false).collect(toList())
    );
    assertEquals(
            asList("Avensis", "Civic", "Focus", "Fusion", "Hilux", "Insight", "M6", "Prius"),
            sortedAttributeMetadata.getDistinctKeys("Alpha", false, "Tango", false).collect(toList())
    );
}
 

@Test
public void testStringMatchesRegexWithIndex() {
    Query<String> query = matchesRegex(selfAttribute(String.class), "f.*");
    IndexedCollection<String> indexedCollection = new ConcurrentIndexedCollection<String>();
    indexedCollection.addAll(asList("foo1", "foo2", "bar", "baz", "car"));
    IndexedCollection<String> collection = indexedCollection;
    collection.addIndex(StandingQueryIndex.onQuery(query));
    ResultSet<String> results = collection.retrieve(query);
    assertEquals(2, results.size());
    assertTrue(results.iterator().hasNext());
}
 

@Test
public void testGetCountForKey() {
    IndexedCollection<Car> cars = createIndexedCollectionOfCars(20);

    // Add an unsorted index on Car.MANUFACTURER (a HashIndex)...
    cars.addIndex(HashIndex.onAttribute(Car.MANUFACTURER));
    MetadataEngine<Car> metadataEngine = cars.getMetadataEngine();

    // Count the number of cars manufactured by BMW...
    AttributeMetadata<String, Car> attributeMetadata = metadataEngine.getAttributeMetadata(Car.MANUFACTURER);
    assertEquals(Integer.valueOf(2), attributeMetadata.getCountForKey("BMW"));
}
 

public static void main(String[] args) {
    IndexedCollection<String> indexedCollection = new ConcurrentIndexedCollection<String>();
    indexedCollection.addAll(Arrays.asList("TEAM", "TEST", "TOAST", "T", "TT"));
    IndexedCollection<String> collection = indexedCollection;
    collection.addIndex(RadixTreeIndex.onAttribute(SELF));
    collection.addIndex(ReversedRadixTreeIndex.onAttribute(SELF));

    for (String match : retrieveWildcardMatches(collection, "T", "T")) {
        System.out.println(match); // TOAST, TEST, TT
    }
}
 

@Test
public void testGetDistinctKeysAndCounts() {
    IndexedCollection<Car> collection = new ConcurrentIndexedCollection<Car>();
    KeyStatisticsIndex<String, Car> MODEL_INDEX = HashIndex.onAttribute(Car.MODEL);
    collection.addIndex(MODEL_INDEX);

    collection.addAll(CarFactory.createCollectionOfCars(20));

    Set<String> distinctModels = setOf(MODEL_INDEX.getDistinctKeys(noQueryOptions()));
    Assert.assertEquals(setOf("Accord", "Avensis", "Civic", "Focus", "Fusion", "Hilux", "Insight", "M6", "Prius", "Taurus"), distinctModels);
    for (String model : distinctModels) {
        Assert.assertEquals(Integer.valueOf(2), MODEL_INDEX.getCountForKey(model, noQueryOptions()));
    }
}
 

/**
 * Tests a configuration where the collection is stored off-heap, one index is on-disk, and one index is on-heap.
 */
@Test
public void testCompositePersistence_EndToEnd() {
    OffHeapPersistence<Car, Integer> offHeapPersistence = OffHeapPersistence.onPrimaryKey(Car.CAR_ID);
    DiskPersistence<Car, Integer> diskPersistence = DiskPersistence.onPrimaryKey(Car.CAR_ID);
    IndexedCollection<Car> collection = new ConcurrentIndexedCollection<Car>(CompositePersistence.of(
            offHeapPersistence,
            diskPersistence,
            singletonList(OnHeapPersistence.onPrimaryKey(Car.CAR_ID))
    ));

    collection.addIndex(DiskIndex.onAttribute(Car.MANUFACTURER));
    collection.addIndex(OffHeapIndex.onAttribute(Car.MODEL));
    collection.addIndex(NavigableIndex.onAttribute(Car.PRICE));

    collection.addAll(CarFactory.createCollectionOfCars(1000));

    ResultSet<Car> results = null;
    try {
        results = collection.retrieve(
                and(
                        or(
                                equal(Car.MANUFACTURER, "Ford"),
                                equal(Car.MODEL, "Avensis")
                        ),
                        lessThan(Car.PRICE, 6000.0)
                )
        );
        Assert.assertEquals(300, results.size());

        Assert.assertTrue(offHeapPersistence.getBytesUsed() > 4096); // example: 163840
        Assert.assertTrue(diskPersistence.getBytesUsed() > 4096); // example: 30720
    }
    finally {
        CloseableRequestResources.closeQuietly(results);
        collection.clear();
        offHeapPersistence.close();
        diskPersistence.getFile().delete();
    }
}
 

@Test
public void testIndexQuantization_LastBucket() {
    IndexedCollection<Car> collection = new ConcurrentIndexedCollection<Car>();
    collection.addIndex(NavigableIndex.withQuantizerOnAttribute(IntegerQuantizer.withCompressionFactor(10), Car.CAR_ID));
    collection.addAll(CarFactory.createCollectionOfCars(100));

    // Merge cost should be 10, because objects matching this query are in a single bucket...
    assertEquals(10, collection.retrieve(between(Car.CAR_ID, 96, 98)).getMergeCost());

    // 3 objects match the query...
    assertEquals(3, collection.retrieve(between(Car.CAR_ID, 96, 98)).size());

    List<Integer> carIdsFound = retrieveCarIds(collection, between(Car.CAR_ID, 96, 98));
    assertEquals(asList(96, 97, 98), carIdsFound);
}
 

@Test
public void testMapFunctionality() {
    IndexedCollection<Map> cars = new ConcurrentIndexedCollection<Map>();

    cars.addIndex(HashIndex.onAttribute(COLOR));
    cars.addIndex(NavigableIndex.onAttribute(DOORS));
    cars.addIndex(RadixTreeIndex.onAttribute(MODEL));
    cars.addIndex(ReversedRadixTreeIndex.onAttribute(MODEL));
    cars.addIndex(InvertedRadixTreeIndex.onAttribute(MODEL));
    cars.addIndex(SuffixTreeIndex.onAttribute(MODEL));

    cars.add(buildNewCar(1, "Ford",   "Focus",  Car.Color.BLUE,  5, 9000.50, Collections.<String>emptyList()));
    cars.add(buildNewCar(2, "Ford",   "Fiesta", Car.Color.BLUE,  2, 5000.00, Collections.<String>emptyList()));
    cars.add(buildNewCar(3, "Ford",   "F-150",  Car.Color.RED,   2, 9500.00, Collections.<String>emptyList()));
    cars.add(buildNewCar(4, "Honda",  "Civic",  Car.Color.RED,   5, 5000.00, Collections.<String>emptyList()));
    cars.add(buildNewCar(5, "Toyota", "Prius",  Car.Color.BLACK, 3, 9700.00, Collections.<String>emptyList()));

    // Ford cars...
    assertThat(carIdsIn(cars.retrieve(equal(MANUFACTURER, "Ford"))), is(setOf(1, 2, 3)));

    // 3-door cars...
    assertThat(carIdsIn(cars.retrieve(equal(DOORS, 3))), is(setOf(5)));

    // 2 or 3-door cars...
    assertThat(carIdsIn(cars.retrieve(between(DOORS, 2, 3))), is(setOf(2, 3, 5)));

    // 2 or 5-door cars...
    assertThat(carIdsIn(cars.retrieve(in(DOORS, 2, 5))), is(setOf(1, 2, 3, 4)));

    // Blue Ford cars...
    assertThat(carIdsIn(cars.retrieve(and(equal(COLOR, Car.Color.BLUE),
            equal(MANUFACTURER, "Ford")))), is(setOf(1, 2)));

    // NOT 3-door cars...
    assertThat(carIdsIn(cars.retrieve(not(equal(DOORS, 3)))),
            is(setOf(1, 2, 3, 4)));

    // Cars which have 5 doors and which are not red...
    assertThat(carIdsIn(cars.retrieve(and(equal(DOORS, 5), not(equal(COLOR, Car.Color.RED))))), is(setOf(1)));

    // Cars whose model starts with 'F'...
    assertThat(carIdsIn(cars.retrieve(startsWith(MODEL, "F"))), is(setOf(1, 2, 3)));

    // Cars whose model ends with 's'...
    assertThat(carIdsIn(cars.retrieve(endsWith(MODEL, "s"))), is(setOf(1, 5)));

    // Cars whose model contains 'i'...
    assertThat(carIdsIn(cars.retrieve(contains(MODEL, "i"))), is(setOf(2, 4, 5)));

    // Cars whose model is contained in 'Banana, Focus, Civic, Foobar'...
    assertThat(carIdsIn(cars.retrieve(isContainedIn(MODEL, "Banana, Focus, Civic, Foobar"))), is(setOf(1, 4)));

    // NOT 3-door cars, sorted by doors ascending...
    assertThat(
            carIdsIn(cars.retrieve(not(equal(DOORS, 3)), queryOptions(orderBy(ascending(DOORS), ascending(MODEL))))).toString(),
            is(equalTo(setOf(3, 2, 4, 1).toString()))
    );

    // NOT 3-door cars, sorted by doors ascending then price descending...
    assertThat(
            carIdsIn(
                    cars.retrieve(
                            not(equal(DOORS, 3)),
                            queryOptions(
                                    orderBy(ascending(DOORS),
                                            descending(PRICE))
                            )
                    )
            ),
            is(equalTo(setOf(3, 2, 1, 4)))
    );
}