org.apache.cassandra.db.filter.IDiskAtomFilter Java Examples

public boolean isFilterFullyCoveredBy(IDiskAtomFilter filter, ColumnFamily cachedCf, long now)
{
    // We can use the cached value only if we know that no data it doesn't contain could be covered
    // by the query filter, that is if:
    //   1) either the whole partition is cached
    //   2) or we can ensure than any data the filter selects are in the cached partition

    // When counting rows to decide if the whole row is cached, we should be careful with expiring
    // columns: if we use a timestamp newer than the one that was used when populating the cache, we might
    // end up deciding the whole partition is cached when it's really not (just some rows expired since the
    // cf was cached). This is the reason for Integer.MIN_VALUE below.
    boolean wholePartitionCached = cachedCf.liveCQL3RowCount(Integer.MIN_VALUE) < metadata.getCaching().rowCache.rowsToCache;

    // Contrarily to the "wholePartitionCached" check above, we do want isFullyCoveredBy to take the
    // timestamp of the query into account when dealing with expired columns. Otherwise, we could think
    // the cached partition has enough live rows to satisfy the filter when it doesn't because some
    // are now expired.
    return wholePartitionCached || filter.isFullyCoveredBy(cachedCf, now);
}
 

private Composite makePrefix(CompositesIndex index, ByteBuffer key, ExtendedFilter filter, boolean isStart)
{
    if (key.remaining() == 0)
        return Composites.EMPTY;

    Composite prefix;
    IDiskAtomFilter columnFilter = filter.columnFilter(key);
    if (columnFilter instanceof SliceQueryFilter)
    {
        SliceQueryFilter sqf = (SliceQueryFilter)columnFilter;
        Composite columnName = isStart ? sqf.start() : sqf.finish();
        prefix = columnName.isEmpty() ? index.getIndexComparator().make(key) : index.makeIndexColumnPrefix(key, columnName);
    }
    else
    {
        prefix = index.getIndexComparator().make(key);
    }
    return isStart ? prefix.start() : prefix.end();
}
 

public ExtendedFilter makeExtendedFilter(AbstractBounds<RowPosition> range,
                                         IDiskAtomFilter columnFilter,
                                         List<IndexExpression> rowFilter,
                                         int maxResults,
                                         boolean countCQL3Rows,
                                         boolean isPaging,
                                         long timestamp)
{
    DataRange dataRange;
    if (isPaging)
    {
        assert columnFilter instanceof SliceQueryFilter;
        SliceQueryFilter sfilter = (SliceQueryFilter)columnFilter;
        assert sfilter.slices.length == 1;
        // create a new SliceQueryFilter that selects all cells, but pass the original slice start and finish
        // through to DataRange.Paging to be used on the first and last partitions
        SliceQueryFilter newFilter = new SliceQueryFilter(ColumnSlice.ALL_COLUMNS_ARRAY, sfilter.isReversed(), sfilter.count);
        dataRange = new DataRange.Paging(range, newFilter, sfilter.start(), sfilter.finish(), metadata.comparator);
    }
    else
    {
        dataRange = new DataRange(range, columnFilter);
    }
    return ExtendedFilter.create(this, dataRange, rowFilter, maxResults, countCQL3Rows, timestamp);
}
 

public RangeSliceCommand deserialize(DataInput in, int version) throws IOException
{
    String keyspace = in.readUTF();
    String columnFamily = in.readUTF();
    long timestamp = in.readLong();

    CFMetaData metadata = Schema.instance.getCFMetaData(keyspace, columnFamily);

    IDiskAtomFilter predicate = metadata.comparator.diskAtomFilterSerializer().deserialize(in, version);

    List<IndexExpression> rowFilter;
    int filterCount = in.readInt();
    rowFilter = new ArrayList<>(filterCount);
    for (int i = 0; i < filterCount; i++)
    {
        rowFilter.add(IndexExpression.readFrom(in));
    }
    AbstractBounds<RowPosition> range = AbstractBounds.serializer.deserialize(in, version).toRowBounds();

    int maxResults = in.readInt();
    boolean countCQL3Rows = in.readBoolean();
    boolean isPaging = in.readBoolean();
    return new RangeSliceCommand(keyspace, columnFamily, timestamp, predicate, range, rowFilter, maxResults, countCQL3Rows, isPaging);
}
 

protected AbstractQueryPager(ConsistencyLevel consistencyLevel,
                             int toFetch,
                             boolean localQuery,
                             CFMetaData cfm,
                             IDiskAtomFilter columnFilter,
                             long timestamp)
{
    this.consistencyLevel = consistencyLevel;
    this.localQuery = localQuery;

    this.cfm = cfm;
    this.columnFilter = columnFilter;
    this.timestamp = timestamp;

    this.remaining = toFetch;
}
 

private static List<Row> getIndexed(ColumnFamilyStore store, IDiskAtomFilter columnFilter, DecoratedKey startKey, int maxResults, IndexExpression... expressions)
{
    IPartitioner p = StorageService.getPartitioner();
    AbstractBounds<RowPosition> bounds;

    if (startKey == null)
    {
        bounds = new Range<>(p.getMinimumToken(), p.getMinimumToken()).toRowBounds();
    }
    else
    {
        bounds = new Bounds<>(startKey, p.getMinimumToken().maxKeyBound(p));
    }

    return store.indexManager.search(ExtendedFilter.create(store,
                                     new DataRange(bounds, columnFilter),
                                     Arrays.asList(expressions),
                                     maxResults,
                                     false,
                                     System.currentTimeMillis()));
}
 

protected AbstractQueryPager(ConsistencyLevel consistencyLevel,
                             int toFetch,
                             boolean localQuery,
                             String keyspace,
                             String columnFamily,
                             IDiskAtomFilter columnFilter,
                             long timestamp)
{
    this(consistencyLevel, toFetch, localQuery, Schema.instance.getCFMetaData(keyspace, columnFamily), columnFilter, timestamp);
}
 

private Map<ByteBuffer, List<ColumnOrSuperColumn>> multigetSliceInternal(String keyspace,
                                                                         List<ByteBuffer> keys,
                                                                         ColumnParent column_parent,
                                                                         long timestamp,
                                                                         SlicePredicate predicate,
                                                                         ConsistencyLevel consistency_level,
                                                                         ClientState cState)
throws org.apache.cassandra.exceptions.InvalidRequestException, UnavailableException, TimedOutException
{
    CFMetaData metadata = ThriftValidation.validateColumnFamily(keyspace, column_parent.column_family);
    ThriftValidation.validateColumnParent(metadata, column_parent);
    ThriftValidation.validatePredicate(metadata, column_parent, predicate);

    org.apache.cassandra.db.ConsistencyLevel consistencyLevel = ThriftConversion.fromThrift(consistency_level);
    consistencyLevel.validateForRead(keyspace);

    List<ReadCommand> commands = new ArrayList<ReadCommand>(keys.size());
    IDiskAtomFilter filter = toInternalFilter(metadata, column_parent, predicate);

    for (ByteBuffer key: keys)
    {
        ThriftValidation.validateKey(metadata, key);
        // Note that we should not share a slice filter amongst the command, due to SliceQueryFilter not  being immutable
        // due to its columnCounter used by the lastCounted() method (also see SelectStatement.getSliceCommands)
        commands.add(ReadCommand.create(keyspace, key, column_parent.getColumn_family(), timestamp, filter.cloneShallow()));
    }

    return getSlice(commands, column_parent.isSetSuper_column(), consistencyLevel, cState);
}
 

public RangeSliceCommand(String keyspace,
                         String columnFamily,
                         long timestamp,
                         IDiskAtomFilter predicate,
                         AbstractBounds<RowPosition> range,
                         int maxResults)
{
    this(keyspace, columnFamily, timestamp, predicate, range, null, maxResults, false, false);
}
 

public RangeSliceCommand(String keyspace,
                         String columnFamily,
                         long timestamp,
                         IDiskAtomFilter predicate,
                         AbstractBounds<RowPosition> range,
                         List<IndexExpression> row_filter,
                         int maxResults)
{
    this(keyspace, columnFamily, timestamp, predicate, range, row_filter, maxResults, false, false);
}
 

public RangeSliceCommand(String keyspace,
                         String columnFamily,
                         long timestamp,
                         IDiskAtomFilter predicate,
                         AbstractBounds<RowPosition> range,
                         List<IndexExpression> rowFilter,
                         int maxResults,
                         boolean countCQL3Rows,
                         boolean isPaging)
{
    super(keyspace, columnFamily, timestamp, range, predicate, rowFilter);
    this.maxResults = maxResults;
    this.countCQL3Rows = countCQL3Rows;
    this.isPaging = isPaging;
}
 

public long serializedSize(RangeSliceCommand rsc, int version)
{
    long size = TypeSizes.NATIVE.sizeof(rsc.keyspace);
    size += TypeSizes.NATIVE.sizeof(rsc.columnFamily);
    size += TypeSizes.NATIVE.sizeof(rsc.timestamp);

    CFMetaData metadata = Schema.instance.getCFMetaData(rsc.keyspace, rsc.columnFamily);

    IDiskAtomFilter filter = rsc.predicate;

    size += metadata.comparator.diskAtomFilterSerializer().serializedSize(filter, version);

    if (rsc.rowFilter == null)
    {
        size += TypeSizes.NATIVE.sizeof(0);
    }
    else
    {
        size += TypeSizes.NATIVE.sizeof(rsc.rowFilter.size());
        for (IndexExpression expr : rsc.rowFilter)
        {
            size += TypeSizes.NATIVE.sizeofWithShortLength(expr.column);
            size += TypeSizes.NATIVE.sizeof(expr.operator.ordinal());
            size += TypeSizes.NATIVE.sizeofWithShortLength(expr.value);
        }
    }
    size += AbstractBounds.serializer.serializedSize(rsc.keyRange, version);
    size += TypeSizes.NATIVE.sizeof(rsc.maxResults);
    size += TypeSizes.NATIVE.sizeof(rsc.countCQL3Rows);
    size += TypeSizes.NATIVE.sizeof(rsc.isPaging);
    return size;
}
 

public static ReadCommand create(String ksName, ByteBuffer key, String cfName, long timestamp, IDiskAtomFilter filter)
{
    if (filter instanceof SliceQueryFilter)
        return new SliceFromReadCommand(ksName, key, cfName, timestamp, (SliceQueryFilter)filter);
    else
        return new SliceByNamesReadCommand(ksName, key, cfName, timestamp, (NamesQueryFilter)filter);
}
 

@VisibleForTesting
public List<Row> getRangeSlice(final AbstractBounds<RowPosition> range,
                               List<IndexExpression> rowFilter,
                               IDiskAtomFilter columnFilter,
                               int maxResults)
{
    return getRangeSlice(range, rowFilter, columnFilter, maxResults, System.currentTimeMillis());
}
 

public List<Row> getRangeSlice(final AbstractBounds<RowPosition> range,
                               List<IndexExpression> rowFilter,
                               IDiskAtomFilter columnFilter,
                               int maxResults,
                               long now)
{
    return getRangeSlice(makeExtendedFilter(range, columnFilter, rowFilter, maxResults, false, false, now));
}
 

public List<Row> getRangeSlice(AbstractBounds<RowPosition> range,
                               List<IndexExpression> rowFilter,
                               IDiskAtomFilter columnFilter,
                               int maxResults,
                               long now,
                               boolean countCQL3Rows,
                               boolean isPaging)
{
    return getRangeSlice(makeExtendedFilter(range, columnFilter, rowFilter, maxResults, countCQL3Rows, isPaging, now));
}
 

@VisibleForTesting
public List<Row> search(AbstractBounds<RowPosition> range,
                        List<IndexExpression> clause,
                        IDiskAtomFilter dataFilter,
                        int maxResults)
{
    return search(range, clause, dataFilter, maxResults, System.currentTimeMillis());
}
 

public List<Row> search(AbstractBounds<RowPosition> range,
                        List<IndexExpression> clause,
                        IDiskAtomFilter dataFilter,
                        int maxResults,
                        long now)
{
    return search(makeExtendedFilter(range, dataFilter, clause, maxResults, false, false, now));
}
 

public static List<Row> getRangeSlice(ColumnFamilyStore cfs, ByteBuffer superColumn)
{
    IDiskAtomFilter filter = superColumn == null
                           ? new IdentityQueryFilter()
                           : new SliceQueryFilter(SuperColumns.startOf(superColumn), SuperColumns.endOf(superColumn), false, Integer.MAX_VALUE);

    Token min = StorageService.getPartitioner().getMinimumToken();
    return cfs.getRangeSlice(new Bounds<Token>(min, min).toRowBounds(), null, filter, 10000);
}
 

/**
 * Create a RangeSliceCommand and run it against the StorageProxy.
 * <p>
 * To match the behavior of the standard Cassandra thrift API endpoint, the
 * {@code nowMillis} argument should be the number of milliseconds since the
 * UNIX Epoch (e.g. System.currentTimeMillis() or equivalent obtained
 * through a {@link TimestampProvider}). This is per
 * {@link org.apache.cassandra.thrift.CassandraServer#get_range_slices(ColumnParent, SlicePredicate, KeyRange, ConsistencyLevel)},
 * which passes the server's System.currentTimeMillis() to the
 * {@code RangeSliceCommand} constructor.
 */
private List<Row> getKeySlice(Token start,
                              Token end,
                              @Nullable SliceQuery sliceQuery,
                              int pageSize,
                              long nowMillis) throws BackendException {
    IPartitioner partitioner = StorageService.getPartitioner();

    SliceRange columnSlice = new SliceRange();
    if (sliceQuery == null) {
        columnSlice.setStart(ArrayUtils.EMPTY_BYTE_ARRAY)
                .setFinish(ArrayUtils.EMPTY_BYTE_ARRAY)
                .setCount(5);
    } else {
        columnSlice.setStart(sliceQuery.getSliceStart().asByteBuffer())
                .setFinish(sliceQuery.getSliceEnd().asByteBuffer())
                .setCount(sliceQuery.hasLimit() ? sliceQuery.getLimit() : Integer.MAX_VALUE);
    }
    /* Note: we need to fetch columns for each row as well to remove "range ghosts" */
    SlicePredicate predicate = new SlicePredicate().setSlice_range(columnSlice);

    // DAVID CASSANDRA
    // Old cassandra code did not use partitioner anyway in this call...so new code removed it as a parmaeter
    // RowPosition startPosition = start.minKeyBound(partitioner);
    RowPosition startPosition = start.minKeyBound();
    // DAVID CASSANDRA
    // RowPosition endPosition = end.minKeyBound(partitioner);
    RowPosition endPosition = end.minKeyBound();

    List<Row> rows;

    try {
        CFMetaData cfm = Schema.instance.getCFMetaData(keyspace, columnFamily);
        IDiskAtomFilter filter = ThriftValidation.asIFilter(predicate, cfm, null);

        RangeSliceCommand cmd = new RangeSliceCommand(keyspace, columnFamily, nowMillis, filter, new Bounds<RowPosition>(startPosition, endPosition), pageSize);

        rows = StorageProxy.getRangeSlice(cmd, ConsistencyLevel.QUORUM);
    } catch (Exception e) {
        throw new PermanentBackendException(e);
    }

    return rows;
}
 

public IDiskAtomFilter readFilter()
{
    return expected == null || expected.isEmpty()
         ? new SliceQueryFilter(ColumnSlice.ALL_COLUMNS_ARRAY, false, 1)
         : new NamesQueryFilter(ImmutableSortedSet.copyOf(expected.getComparator(), expected.getColumnNames()));
}
 

@Test
public void testIndexUpdateOverwritingExpiringColumns() throws Exception
{
    // see CASSANDRA-7268
    Keyspace keyspace = Keyspace.open("Keyspace2");

    // create a row and update the birthdate value with an expiring column
    Mutation rm;
    rm = new Mutation("Keyspace2", ByteBufferUtil.bytes("k100"));
    rm.add("Indexed1", cellname("birthdate"), ByteBufferUtil.bytes(100L), 1, 1000);
    rm.apply();

    IndexExpression expr = new IndexExpression(ByteBufferUtil.bytes("birthdate"), Operator.EQ, ByteBufferUtil.bytes(100L));
    List<IndexExpression> clause = Arrays.asList(expr);
    IDiskAtomFilter filter = new IdentityQueryFilter();
    Range<RowPosition> range = Util.range("", "");
    List<Row> rows = keyspace.getColumnFamilyStore("Indexed1").search(range, clause, filter, 100);
    assertEquals(1, rows.size());

    // requires a 1s sleep because we calculate local expiry time as (now() / 1000) + ttl
    TimeUnit.SECONDS.sleep(1);

    // now overwrite with the same name/value/ttl, but the local expiry time will be different
    rm = new Mutation("Keyspace2", ByteBufferUtil.bytes("k100"));
    rm.add("Indexed1", cellname("birthdate"), ByteBufferUtil.bytes(100L), 1, 1000);
    rm.apply();

    rows = keyspace.getColumnFamilyStore("Indexed1").search(range, clause, filter, 100);
    assertEquals(1, rows.size());

    // check that modifying the indexed value using the same timestamp behaves as expected
    rm = new Mutation("Keyspace2", ByteBufferUtil.bytes("k101"));
    rm.add("Indexed1", cellname("birthdate"), ByteBufferUtil.bytes(101L), 1, 1000);
    rm.apply();

    expr = new IndexExpression(ByteBufferUtil.bytes("birthdate"), Operator.EQ, ByteBufferUtil.bytes(101L));
    clause = Arrays.asList(expr);
    rows = keyspace.getColumnFamilyStore("Indexed1").search(range, clause, filter, 100);
    assertEquals(1, rows.size());

    TimeUnit.SECONDS.sleep(1);
    rm = new Mutation("Keyspace2", ByteBufferUtil.bytes("k101"));
    rm.add("Indexed1", cellname("birthdate"), ByteBufferUtil.bytes(102L), 1, 1000);
    rm.apply();
    // search for the old value
    rows = keyspace.getColumnFamilyStore("Indexed1").search(range, clause, filter, 100);
    assertEquals(0, rows.size());
    // and for the new
    expr = new IndexExpression(ByteBufferUtil.bytes("birthdate"), Operator.EQ, ByteBufferUtil.bytes(102L));
    clause = Arrays.asList(expr);
    rows = keyspace.getColumnFamilyStore("Indexed1").search(range, clause, filter, 100);
    assertEquals(1, rows.size());
}
 

@Test
public void rangeTombstoneFilteringTest() throws Exception
{
    CompactionManager.instance.disableAutoCompaction();
    Keyspace keyspace = Keyspace.open(KSNAME);
    ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(CFNAME);

    // Inserting data
    String key = "k111";
    Mutation rm;
    ColumnFamily cf;

    rm = new Mutation(KSNAME, ByteBufferUtil.bytes(key));
    for (int i = 0; i < 40; i += 2)
        add(rm, i, 0);
    rm.apply();

    rm = new Mutation(KSNAME, ByteBufferUtil.bytes(key));
    cf = rm.addOrGet(CFNAME);
    delete(cf, 5, 10, 1);
    rm.apply();

    rm = new Mutation(KSNAME, ByteBufferUtil.bytes(key));
    cf = rm.addOrGet(CFNAME);
    delete(cf, 15, 20, 2);
    rm.apply();

    cf = cfs.getColumnFamily(QueryFilter.getSliceFilter(dk(key), CFNAME, b(11), b(14), false, Integer.MAX_VALUE, System.currentTimeMillis()));
    Collection<RangeTombstone> rt = rangeTombstones(cf);
    assertEquals(0, rt.size());

    cf = cfs.getColumnFamily(QueryFilter.getSliceFilter(dk(key), CFNAME, b(11), b(15), false, Integer.MAX_VALUE, System.currentTimeMillis()));
    rt = rangeTombstones(cf);
    assertEquals(1, rt.size());

    cf = cfs.getColumnFamily(QueryFilter.getSliceFilter(dk(key), CFNAME, b(20), b(25), false, Integer.MAX_VALUE, System.currentTimeMillis()));
    rt = rangeTombstones(cf);
    assertEquals(1, rt.size());

    cf = cfs.getColumnFamily(QueryFilter.getSliceFilter(dk(key), CFNAME, b(12), b(25), false, Integer.MAX_VALUE, System.currentTimeMillis()));
    rt = rangeTombstones(cf);
    assertEquals(1, rt.size());

    cf = cfs.getColumnFamily(QueryFilter.getSliceFilter(dk(key), CFNAME, b(25), b(35), false, Integer.MAX_VALUE, System.currentTimeMillis()));
    rt = rangeTombstones(cf);
    assertEquals(0, rt.size());

    cf = cfs.getColumnFamily(QueryFilter.getSliceFilter(dk(key), CFNAME, b(1), b(40), false, Integer.MAX_VALUE, System.currentTimeMillis()));
    rt = rangeTombstones(cf);
    assertEquals(2, rt.size());

    cf = cfs.getColumnFamily(QueryFilter.getSliceFilter(dk(key), CFNAME, b(7), b(17), false, Integer.MAX_VALUE, System.currentTimeMillis()));
    rt = rangeTombstones(cf);
    assertEquals(2, rt.size());

    cf = cfs.getColumnFamily(QueryFilter.getSliceFilter(dk(key), CFNAME, b(5), b(20), false, Integer.MAX_VALUE, System.currentTimeMillis()));
    rt = rangeTombstones(cf);
    assertEquals(2, rt.size());

    cf = cfs.getColumnFamily(QueryFilter.getSliceFilter(dk(key), CFNAME, b(5), b(15), false, Integer.MAX_VALUE, System.currentTimeMillis()));
    rt = rangeTombstones(cf);
    assertEquals(2, rt.size());

    cf = cfs.getColumnFamily(QueryFilter.getSliceFilter(dk(key), CFNAME, b(1), b(2), false, Integer.MAX_VALUE, System.currentTimeMillis()));
    rt = rangeTombstones(cf);
    assertEquals(0, rt.size());

    cf = cfs.getColumnFamily(QueryFilter.getSliceFilter(dk(key), CFNAME, b(1), b(5), false, Integer.MAX_VALUE, System.currentTimeMillis()));
    rt = rangeTombstones(cf);
    assertEquals(1, rt.size());

    cf = cfs.getColumnFamily(QueryFilter.getSliceFilter(dk(key), CFNAME, b(1), b(10), false, Integer.MAX_VALUE, System.currentTimeMillis()));
    rt = rangeTombstones(cf);
    assertEquals(1, rt.size());

    cf = cfs.getColumnFamily(QueryFilter.getSliceFilter(dk(key), CFNAME, b(5), b(6), false, Integer.MAX_VALUE, System.currentTimeMillis()));
    rt = rangeTombstones(cf);
    assertEquals(1, rt.size());

    cf = cfs.getColumnFamily(QueryFilter.getSliceFilter(dk(key), CFNAME, b(17), b(20), false, Integer.MAX_VALUE, System.currentTimeMillis()));
    rt = rangeTombstones(cf);
    assertEquals(1, rt.size());

    cf = cfs.getColumnFamily(QueryFilter.getSliceFilter(dk(key), CFNAME, b(17), b(18), false, Integer.MAX_VALUE, System.currentTimeMillis()));
    rt = rangeTombstones(cf);
    assertEquals(1, rt.size());

    ColumnSlice[] slices = new ColumnSlice[]{new ColumnSlice( b(1), b(10)), new ColumnSlice( b(16), b(20))};
    IDiskAtomFilter sqf = new SliceQueryFilter(slices, false, Integer.MAX_VALUE);
    cf = cfs.getColumnFamily( new QueryFilter(dk(key), CFNAME, sqf, System.currentTimeMillis()) );
    rt = rangeTombstones(cf);
    assertEquals(2, rt.size());
}
 

@Test
public void testDeleteCompositeIndex() throws Exception
{
    String keySpace = "Keyspace2";
    String cfName = "Indexed3"; // has gcGrace 0

    Keyspace keyspace = Keyspace.open(keySpace);
    ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(cfName);
    cfs.truncateBlocking();

    ByteBuffer rowKey = ByteBufferUtil.bytes("k1");
    ByteBuffer clusterKey = ByteBufferUtil.bytes("ck1");
    ByteBuffer colName = ByteBufferUtil.bytes("col1");

    CellNameType baseComparator = cfs.getComparator();
    CellName compositeName = baseComparator.makeCellName(clusterKey, colName);

    ByteBuffer val1 = ByteBufferUtil.bytes("v2");

    // Insert indexed value.
    Mutation rm;
    rm = new Mutation(keySpace, rowKey);
    rm.add(cfName, compositeName, val1, 0);
    rm.apply();

    // Now delete the value and flush too.
    rm = new Mutation(keySpace, rowKey);
    rm.delete(cfName, 1);
    rm.apply();

    // We want the data to be gcable, but even if gcGrace == 0, we still need to wait 1 second
    // since we won't gc on a tie.
    try { Thread.sleep(1000); } catch (Exception e) {}

    // Read the index and we check we do get no value (and no NPE)
    // Note: the index will return the entry because it hasn't been deleted (we
    // haven't read yet nor compacted) but the data read itself will return null
    IndexExpression expr = new IndexExpression(colName, Operator.EQ, val1);
    List<IndexExpression> clause = Arrays.asList(expr);
    IDiskAtomFilter filter = new IdentityQueryFilter();
    Range<RowPosition> range = Util.range("", "");
    List<Row> rows = keyspace.getColumnFamilyStore(cfName).search(range, clause, filter, 100);
    assertEquals(0, rows.size());
}
 

@Test
public void testDeleteOfInconsistentValuesInKeysIndex() throws Exception
{
    String keySpace = "Keyspace2";
    String cfName = "Indexed1";

    Keyspace keyspace = Keyspace.open(keySpace);
    ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(cfName);
    cfs.truncateBlocking();

    ByteBuffer rowKey = ByteBufferUtil.bytes("k1");
    CellName colName = cellname("birthdate"); 
    ByteBuffer val1 = ByteBufferUtil.bytes(1L);
    ByteBuffer val2 = ByteBufferUtil.bytes(2L);

    // create a row and update the "birthdate" value, test that the index query fetches this version
    Mutation rm;
    rm = new Mutation(keySpace, rowKey);
    rm.add(cfName, colName, val1, 0);
    rm.apply();
    IndexExpression expr = new IndexExpression(ByteBufferUtil.bytes("birthdate"), Operator.EQ, val1);
    List<IndexExpression> clause = Arrays.asList(expr);
    IDiskAtomFilter filter = new IdentityQueryFilter();
    Range<RowPosition> range = Util.range("", "");
    List<Row> rows = keyspace.getColumnFamilyStore(cfName).search(range, clause, filter, 100);
    assertEquals(1, rows.size());

    // force a flush, so our index isn't being read from a memtable
    keyspace.getColumnFamilyStore(cfName).forceBlockingFlush();

    // now apply another update, but force the index update to be skipped
    rm = new Mutation(keySpace, rowKey);
    rm.add(cfName, colName, val2, 1);
    keyspace.apply(rm, true, false);

    // Now searching the index for either the old or new value should return 0 rows
    // because the new value was not indexed and the old value should be ignored
    // (and in fact purged from the index cf).
    // first check for the old value
    rows = keyspace.getColumnFamilyStore(cfName).search(range, clause, filter, 100);
    assertEquals(0, rows.size());
    // now check for the updated value
    expr = new IndexExpression(ByteBufferUtil.bytes("birthdate"), Operator.EQ, val2);
    clause = Arrays.asList(expr);
    filter = new IdentityQueryFilter();
    range = Util.range("", "");
    rows = keyspace.getColumnFamilyStore(cfName).search(range, clause, filter, 100);
    assertEquals(0, rows.size());

    // now, reset back to the original value, still skipping the index update, to
    // make sure the value was expunged from the index when it was discovered to be inconsistent
    rm = new Mutation(keySpace, rowKey);
    rm.add(cfName, colName, ByteBufferUtil.bytes(1L), 3);
    keyspace.apply(rm, true, false);

    expr = new IndexExpression(ByteBufferUtil.bytes("birthdate"), Operator.EQ, ByteBufferUtil.bytes(1L));
    clause = Arrays.asList(expr);
    filter = new IdentityQueryFilter();
    range = Util.range("", "");
    rows = keyspace.getColumnFamilyStore(cfName).search(range, clause, filter, 100);
    assertEquals(0, rows.size());
}
 

@Test
public void testDeleteOfInconsistentValuesFromCompositeIndex() throws Exception
{
    String keySpace = "Keyspace2";
    String cfName = "Indexed2";

    Keyspace keyspace = Keyspace.open(keySpace);
    ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(cfName);
    cfs.truncateBlocking();

    ByteBuffer rowKey = ByteBufferUtil.bytes("k1");
    ByteBuffer clusterKey = ByteBufferUtil.bytes("ck1");
    ByteBuffer colName = ByteBufferUtil.bytes("col1"); 

    CellNameType baseComparator = cfs.getComparator();
    CellName compositeName = baseComparator.makeCellName(clusterKey, colName);

    ByteBuffer val1 = ByteBufferUtil.bytes("v1");
    ByteBuffer val2 = ByteBufferUtil.bytes("v2");

    // create a row and update the author value
    Mutation rm;
    rm = new Mutation(keySpace, rowKey);
    rm.add(cfName, compositeName, val1, 0);
    rm.apply();

    // test that the index query fetches this version
    IndexExpression expr = new IndexExpression(colName, Operator.EQ, val1);
    List<IndexExpression> clause = Arrays.asList(expr);
    IDiskAtomFilter filter = new IdentityQueryFilter();
    Range<RowPosition> range = Util.range("", "");
    List<Row> rows = keyspace.getColumnFamilyStore(cfName).search(range, clause, filter, 100);
    assertEquals(1, rows.size());

    // force a flush and retry the query, so our index isn't being read from a memtable
    keyspace.getColumnFamilyStore(cfName).forceBlockingFlush();
    rows = keyspace.getColumnFamilyStore(cfName).search(range, clause, filter, 100);
    assertEquals(1, rows.size());

    // now apply another update, but force the index update to be skipped
    rm = new Mutation(keySpace, rowKey);
    rm.add(cfName, compositeName, val2, 1);
    keyspace.apply(rm, true, false);

    // Now searching the index for either the old or new value should return 0 rows
    // because the new value was not indexed and the old value should be ignored
    // (and in fact purged from the index cf).
    // first check for the old value
    rows = keyspace.getColumnFamilyStore(cfName).search(range, clause, filter, 100);
    assertEquals(0, rows.size());
    // now check for the updated value
    expr = new IndexExpression(colName, Operator.EQ, val2);
    clause = Arrays.asList(expr);
    filter = new IdentityQueryFilter();
    range = Util.range("", "");
    rows = keyspace.getColumnFamilyStore(cfName).search(range, clause, filter, 100);
    assertEquals(0, rows.size());

    // now, reset back to the original value, still skipping the index update, to
    // make sure the value was expunged from the index when it was discovered to be inconsistent
    rm = new Mutation(keySpace, rowKey);
    rm.add(cfName, compositeName, val1, 2);
    keyspace.apply(rm, true, false);

    expr = new IndexExpression(colName, Operator.EQ, val1);
    clause = Arrays.asList(expr);
    filter = new IdentityQueryFilter();
    range = Util.range("", "");
    rows = keyspace.getColumnFamilyStore(cfName).search(range, clause, filter, 100);
    assertEquals(0, rows.size());
}
 

public int getLiveCount(IDiskAtomFilter filter, long now)
{
    return cf == null ? 0 : filter.getLiveCount(cf, now);
}
 

public IDiskAtomFilter filter()
{
    return filter;
}
 

public IVersionedSerializer<IDiskAtomFilter> diskAtomFilterSerializer()
{
    return diskAtomFilterSerializer;
}
 

/**
 * Get a row iterator over the provided memtables and sstables, between the provided keys
 * and filtered by the queryfilter.
 * @param memtables Memtables pending flush.
 * @param sstables SStables to scan through.
 * @param range The data range to fetch
 * @param cfs
 * @return A row iterator following all the given restrictions
 */
public static CloseableIterator<Row> getIterator(final Iterable<Memtable> memtables,
                                                 final Collection<SSTableReader> sstables,
                                                 final DataRange range,
                                                 final ColumnFamilyStore cfs,
                                                 final long now)
{
    // fetch data from current memtable, historical memtables, and SSTables in the correct order.
    final List<CloseableIterator<OnDiskAtomIterator>> iterators = new ArrayList<>(Iterables.size(memtables) + sstables.size());

    for (Memtable memtable : memtables)
        iterators.add(new ConvertToColumnIterator(range, memtable.getEntryIterator(range.startKey(), range.stopKey())));

    for (SSTableReader sstable : sstables)
        iterators.add(sstable.getScanner(range));

    // reduce rows from all sources into a single row
    return MergeIterator.get(iterators, COMPARE_BY_KEY, new MergeIterator.Reducer<OnDiskAtomIterator, Row>()
    {
        private final int gcBefore = cfs.gcBefore(now);
        private final List<OnDiskAtomIterator> colIters = new ArrayList<>();
        private DecoratedKey key;
        private ColumnFamily returnCF;

        @Override
        protected void onKeyChange()
        {
            this.returnCF = ArrayBackedSortedColumns.factory.create(cfs.metadata, range.columnFilter.isReversed());
        }

        public void reduce(OnDiskAtomIterator current)
        {
            this.colIters.add(current);
            this.key = current.getKey();
            this.returnCF.delete(current.getColumnFamily());
        }

        protected Row getReduced()
        {
            // First check if this row is in the rowCache. If it is and it covers our filter, we can skip the rest
            ColumnFamily cached = cfs.getRawCachedRow(key);
            IDiskAtomFilter filter = range.columnFilter(key.getKey());

            if (cached == null || !cfs.isFilterFullyCoveredBy(filter, cached, now))
            {
                // not cached: collate
                QueryFilter.collateOnDiskAtom(returnCF, colIters, filter, gcBefore, now);
            }
            else
            {
                QueryFilter keyFilter = new QueryFilter(key, cfs.name, filter, now);
                returnCF = cfs.filterColumnFamily(cached, keyFilter);
            }

            Row rv = new Row(key, returnCF);
            colIters.clear();
            key = null;
            return rv;
        }
    });
}