org.apache.cassandra.db.compaction.CompactionManager Java Examples

@VisibleForTesting
protected synchronized void compact()
{
    ArrayList<Descriptor> descriptors = new ArrayList<>();
    for (SSTable sstable : hintStore.getDataTracker().getUncompactingSSTables())
        descriptors.add(sstable.descriptor);

    if (descriptors.isEmpty())
        return;

    try
    {
        CompactionManager.instance.submitUserDefined(hintStore, descriptors, (int) (System.currentTimeMillis() / 1000)).get();
    }
    catch (InterruptedException | ExecutionException e)
    {
        throw new RuntimeException(e);
    }
}
 

/**
 * For CASSANDRA-6892 too, check that for a compact table with one cluster column, we can insert whatever
 * we want as value for the clustering column, including something that would conflict with a CQL column definition.
 */
@Test
public void testValidationCompactStorage() throws Exception
{
    QueryProcessor.process("CREATE TABLE \"Keyspace1\".test_compact_dynamic_columns (a int, b text, c text, PRIMARY KEY (a, b)) WITH COMPACT STORAGE", ConsistencyLevel.ONE);

    Keyspace keyspace = Keyspace.open("Keyspace1");
    ColumnFamilyStore cfs = keyspace.getColumnFamilyStore("test_compact_dynamic_columns");

    QueryProcessor.executeInternal("INSERT INTO \"Keyspace1\".test_compact_dynamic_columns (a, b, c) VALUES (0, 'a', 'foo')");
    QueryProcessor.executeInternal("INSERT INTO \"Keyspace1\".test_compact_dynamic_columns (a, b, c) VALUES (0, 'b', 'bar')");
    QueryProcessor.executeInternal("INSERT INTO \"Keyspace1\".test_compact_dynamic_columns (a, b, c) VALUES (0, 'c', 'boo')");
    cfs.forceBlockingFlush();
    CompactionManager.instance.performScrub(cfs, true);

    // Scrub is silent, but it will remove broken records. So reading everything back to make sure nothing to "scrubbed away"
    UntypedResultSet rs = QueryProcessor.executeInternal("SELECT * FROM \"Keyspace1\".test_compact_dynamic_columns");
    assertEquals(3, rs.size());

    Iterator<UntypedResultSet.Row> iter = rs.iterator();
    assertEquals("foo", iter.next().getString("c"));
    assertEquals("bar", iter.next().getString("c"));
    assertEquals("boo", iter.next().getString("c"));
}
 

/**
 * Tests CASSANDRA-6892 (key aliases being used improperly for validation)
 */
@Test
public void testColumnNameEqualToDefaultKeyAlias() throws ExecutionException, InterruptedException
{
    Keyspace keyspace = Keyspace.open("Keyspace1");
    ColumnFamilyStore cfs = keyspace.getColumnFamilyStore("UUIDKeys");

    ColumnFamily cf = ArrayBackedSortedColumns.factory.create("Keyspace1", "UUIDKeys");
    cf.addColumn(column(CFMetaData.DEFAULT_KEY_ALIAS, "not a uuid", 1L));
    Mutation mutation = new Mutation("Keyspace1", ByteBufferUtil.bytes(UUIDGen.getTimeUUID()), cf);
    mutation.applyUnsafe();
    cfs.forceBlockingFlush();
    CompactionManager.instance.performScrub(cfs, false);

    assertEquals(1, cfs.getSSTables().size());
}
 

@Test
public void testScrubMultiRow() throws ExecutionException, InterruptedException
{
    CompactionManager.instance.disableAutoCompaction();
    Keyspace keyspace = Keyspace.open(KEYSPACE);
    ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(CF);
    cfs.clearUnsafe();

    List<Row> rows;

    // insert data and verify we get it back w/ range query
    fillCF(cfs, 10);
    rows = cfs.getRangeSlice(Util.range("", ""), null, new IdentityQueryFilter(), 1000);
    assertEquals(10, rows.size());

    CompactionManager.instance.performScrub(cfs, false);

    // check data is still there
    rows = cfs.getRangeSlice(Util.range("", ""), null, new IdentityQueryFilter(), 1000);
    assertEquals(10, rows.size());
}
 

@Test
public void testScrubOneRow() throws ExecutionException, InterruptedException
{
    CompactionManager.instance.disableAutoCompaction();
    Keyspace keyspace = Keyspace.open(KEYSPACE);
    ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(CF);
    cfs.clearUnsafe();

    List<Row> rows;

    // insert data and verify we get it back w/ range query
    fillCF(cfs, 1);
    rows = cfs.getRangeSlice(Util.range("", ""), null, new IdentityQueryFilter(), 1000);
    assertEquals(1, rows.size());

    CompactionManager.instance.performScrub(cfs, false);

    // check data is still there
    rows = cfs.getRangeSlice(Util.range("", ""), null, new IdentityQueryFilter(), 1000);
    assertEquals(1, rows.size());
}
 

protected void fillCF(ColumnFamilyStore cfs, int rowsPerSSTable)
{
    CompactionManager.instance.disableAutoCompaction();

    for (int i = 0; i < rowsPerSSTable; i++)
    {
        String key = String.valueOf(i);
        // create a row and update the birthdate value, test that the index query fetches the new version
        Mutation rm;
        rm = new Mutation(KEYSPACE1, ByteBufferUtil.bytes(key));
        rm.add(cfs.name, Util.cellname(COLUMN), VALUE, System.currentTimeMillis());
        rm.applyUnsafe();
    }

    cfs.forceBlockingFlush();
}
 

/**
 * Does a full, blocking rebuild of the indexes specified by columns from the sstables.
 * Does nothing if columns is empty.
 *
 * Caller must acquire and release references to the sstables used here.
 *
 * @param sstables the data to build from
 * @param idxNames the list of columns to index, ordered by comparator
 */
public void maybeBuildSecondaryIndexes(Collection<SSTableReader> sstables, Set<String> idxNames)
{
    if (idxNames.isEmpty())
        return;

    logger.info(String.format("Submitting index build of %s for data in %s",
                              idxNames, StringUtils.join(sstables, ", ")));

    SecondaryIndexBuilder builder = new SecondaryIndexBuilder(baseCfs, idxNames, new ReducingKeyIterator(sstables));
    Future<?> future = CompactionManager.instance.submitIndexBuild(builder);
    FBUtilities.waitOnFuture(future);

    flushIndexesBlocking();

    logger.info("Index build of {} complete", idxNames);
}
 

/**
 * Builds the index using the data in the underlying CFS
 * Blocks till it's complete
 */
protected void buildIndexBlocking()
{
    logger.info(String.format("Submitting index build of %s for data in %s",
            getIndexName(), StringUtils.join(baseCfs.getSSTables(), ", ")));

    try (Refs<SSTableReader> sstables = baseCfs.selectAndReference(ColumnFamilyStore.CANONICAL_SSTABLES).refs)
    {
        SecondaryIndexBuilder builder = new SecondaryIndexBuilder(baseCfs,
                                                                  Collections.singleton(getIndexName()),
                                                                  new ReducingKeyIterator(sstables));
        Future<?> future = CompactionManager.instance.submitIndexBuild(builder);
        FBUtilities.waitOnFuture(future);
        forceBlockingFlush();
        setIndexBuilt();
    }
    logger.info("Index build of {} complete", getIndexName());
}
 

private void cleanup() throws ExecutionException, InterruptedException
{
    ColumnFamilyStore cfs = Keyspace.open(Keyspace.SYSTEM_KS).getColumnFamilyStore(SystemKeyspace.BATCHLOG_CF);
    cfs.forceBlockingFlush();
    Collection<Descriptor> descriptors = new ArrayList<>();
    for (SSTableReader sstr : cfs.getSSTables())
        descriptors.add(sstr.descriptor);
    if (!descriptors.isEmpty()) // don't pollute the logs if there is nothing to compact.
        CompactionManager.instance.submitUserDefined(cfs, descriptors, Integer.MAX_VALUE).get();
}
 

public void testGetPositionsKeyCacheStats() throws IOException, ExecutionException, InterruptedException
{
    Keyspace keyspace = Keyspace.open("Keyspace1");
    ColumnFamilyStore store = keyspace.getColumnFamilyStore("Standard2");
    CacheService.instance.keyCache.setCapacity(1000);

    // insert data and compact to a single sstable
    CompactionManager.instance.disableAutoCompaction();
    for (int j = 0; j < 10; j++)
    {
        ByteBuffer key = ByteBufferUtil.bytes(String.valueOf(j));
        Mutation rm = new Mutation("Keyspace1", key);
        rm.add("Standard2", cellname("0"), ByteBufferUtil.EMPTY_BYTE_BUFFER, j);
        rm.apply();
    }
    store.forceBlockingFlush();
    CompactionManager.instance.performMaximal(store);

    SSTableReader sstable = store.getSSTables().iterator().next();
    sstable.getPosition(k(2), SSTableReader.Operator.EQ);
    assertEquals(0, sstable.getKeyCacheHit());
    assertEquals(1, sstable.getBloomFilterTruePositiveCount());
    sstable.getPosition(k(2), SSTableReader.Operator.EQ);
    assertEquals(1, sstable.getKeyCacheHit());
    assertEquals(2, sstable.getBloomFilterTruePositiveCount());
    sstable.getPosition(k(15), SSTableReader.Operator.EQ);
    assertEquals(1, sstable.getKeyCacheHit());
    assertEquals(2, sstable.getBloomFilterTruePositiveCount());

}
 

@Override
public void startNextKeyspace() {
    final String keyspace = super.nextKeyspace();
    if (keyspace == null) {
        return;
    }

    cleanupFuture = executorService.submit(new Runnable() {
        @Override
        public void run() {
            try {
                LOGGER.info("Starting cleanup on keyspace {}", keyspace);
                keyspaceStarted();
                final List<String> cfNames = checkNotNull(jmxConnect).getColumnFamilyNames(keyspace);
                for (final String cfName : cfNames) {
                    final int status = jmxConnect.getStorageServiceProxy().forceKeyspaceCleanup(keyspace, cfName);
                    CompactionManager.AllSSTableOpStatus s = CompactionManager.AllSSTableOpStatus.SUCCESSFUL;
                    for (final CompactionManager.AllSSTableOpStatus st : CompactionManager.AllSSTableOpStatus.values()) {
                        if (st.statusCode == status) {
                            s = st;
                        }
                    }
                    LOGGER.info("Cleanup of {}.{} returned with {}", keyspace, cfName, s);
                }
                keyspaceFinished(SUCCESS, keyspace);
            } catch (final Exception e) {
                LOGGER.error("Failed to cleanup keyspace " + keyspace, e);
                keyspaceFinished(FAILURE, keyspace);
            } finally {
                startNextKeyspace();
            }
        }
    });

    LOGGER.info("Submitted cleanup for keyspace {}", keyspace);
}
 

public static Future<?> compactAll(ColumnFamilyStore cfs, int gcBefore)
{
    List<Descriptor> descriptors = new ArrayList<>();
    for (SSTableReader sstable : cfs.getSSTables())
        descriptors.add(sstable.descriptor);
    return CompactionManager.instance.submitUserDefined(cfs, descriptors, gcBefore);
}
 

public int upgradeSSTables(String keyspaceName, boolean excludeCurrentVersion, String... columnFamilies) throws IOException, ExecutionException, InterruptedException
{
    CompactionManager.AllSSTableOpStatus status = CompactionManager.AllSSTableOpStatus.SUCCESSFUL;
    for (ColumnFamilyStore cfStore : getValidColumnFamilies(true, true, keyspaceName, columnFamilies))
    {
        CompactionManager.AllSSTableOpStatus oneStatus = cfStore.sstablesRewrite(excludeCurrentVersion);
        if (oneStatus != CompactionManager.AllSSTableOpStatus.SUCCESSFUL)
            status = oneStatus;
    }
    return status.statusCode;
}
 

public int scrub(boolean disableSnapshot, boolean skipCorrupted, String keyspaceName, String... columnFamilies) throws IOException, ExecutionException, InterruptedException
{
    CompactionManager.AllSSTableOpStatus status = CompactionManager.AllSSTableOpStatus.SUCCESSFUL;
    for (ColumnFamilyStore cfStore : getValidColumnFamilies(false, false, keyspaceName, columnFamilies))
    {
        CompactionManager.AllSSTableOpStatus oneStatus = cfStore.scrub(disableSnapshot, skipCorrupted);
        if (oneStatus != CompactionManager.AllSSTableOpStatus.SUCCESSFUL)
            status = oneStatus;
    }
    return status.statusCode;
}
 

@Test
public void testScrubColumnValidation() throws InterruptedException, RequestExecutionException, ExecutionException
{
    QueryProcessor.process("CREATE TABLE \"Keyspace1\".test_compact_static_columns (a bigint, b timeuuid, c boolean static, d text, PRIMARY KEY (a, b))", ConsistencyLevel.ONE);

    Keyspace keyspace = Keyspace.open("Keyspace1");
    ColumnFamilyStore cfs = keyspace.getColumnFamilyStore("test_compact_static_columns");

    QueryProcessor.executeInternal("INSERT INTO \"Keyspace1\".test_compact_static_columns (a, b, c, d) VALUES (123, c3db07e8-b602-11e3-bc6b-e0b9a54a6d93, true, 'foobar')");
    cfs.forceBlockingFlush();
    CompactionManager.instance.performScrub(cfs, false);
}
 

public int forceKeyspaceCleanup(String keyspaceName, String... columnFamilies) throws IOException, ExecutionException, InterruptedException
{
    if (keyspaceName.equals(Keyspace.SYSTEM_KS))
        throw new RuntimeException("Cleanup of the system keyspace is neither necessary nor wise");

    CompactionManager.AllSSTableOpStatus status = CompactionManager.AllSSTableOpStatus.SUCCESSFUL;
    for (ColumnFamilyStore cfStore : getValidColumnFamilies(false, false, keyspaceName, columnFamilies))
    {
        CompactionManager.AllSSTableOpStatus oneStatus = cfStore.forceCleanup();
        if (oneStatus != CompactionManager.AllSSTableOpStatus.SUCCESSFUL)
            status = oneStatus;
    }
    return status.statusCode;
}
 

@Test
public void testCleanupWithNewToken() throws ExecutionException, InterruptedException, UnknownHostException
{
    StorageService.instance.getTokenMetadata().clearUnsafe();

    Keyspace keyspace = Keyspace.open(KEYSPACE1);
    ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(CF2);

    List<Row> rows;

    // insert data and verify we get it back w/ range query
    fillCF(cfs, LOOPS);

    rows = Util.getRangeSlice(cfs);

    assertEquals(LOOPS, rows.size());
    TokenMetadata tmd = StorageService.instance.getTokenMetadata();

    byte[] tk1 = new byte[1], tk2 = new byte[1];
    tk1[0] = 2;
    tk2[0] = 1;
    tmd.updateNormalToken(new BytesToken(tk1), InetAddress.getByName("127.0.0.1"));
    tmd.updateNormalToken(new BytesToken(tk2), InetAddress.getByName("127.0.0.2"));
    CompactionManager.instance.performCleanup(cfs);

    rows = Util.getRangeSlice(cfs);
    assertEquals(0, rows.size());
}
 

private void runCompactionWithRangeTombstoneAndCheckSecondaryIndex() throws Exception
{
    Keyspace table = Keyspace.open(KSNAME);
    ColumnFamilyStore cfs = table.getColumnFamilyStore(CFNAME);
    ByteBuffer key = ByteBufferUtil.bytes("k5");
    ByteBuffer indexedColumnName = ByteBufferUtil.bytes(1);

    cfs.truncateBlocking();
    cfs.disableAutoCompaction();
    cfs.setCompactionStrategyClass(SizeTieredCompactionStrategy.class.getCanonicalName());
    if (cfs.indexManager.getIndexForColumn(indexedColumnName) == null)
    {
        ColumnDefinition cd = ColumnDefinition.regularDef(cfs.metadata, indexedColumnName, cfs.getComparator().asAbstractType(), 0)
                                              .setIndex("test_index", IndexType.CUSTOM, ImmutableMap.of(SecondaryIndex.CUSTOM_INDEX_OPTION_NAME, TestIndex.class.getName()));
        cfs.indexManager.addIndexedColumn(cd);
    }

    TestIndex index = ((TestIndex)cfs.indexManager.getIndexForColumn(indexedColumnName));
    index.resetCounts();

    Mutation rm = new Mutation(KSNAME, key);
    for (int i = 0; i < 10; i++)
        add(rm, i, 0);
    rm.apply();
    cfs.forceBlockingFlush();

    rm = new Mutation(KSNAME, key);
    ColumnFamily cf = rm.addOrGet(CFNAME);
    for (int i = 0; i < 10; i += 2)
        delete(cf, 0, 7, 0);
    rm.apply();
    cfs.forceBlockingFlush();

    // We should have indexed 1 column
    assertEquals(1, index.inserts.size());

    CompactionManager.instance.performMaximal(cfs);

    // compacted down to single sstable
    assertEquals(1, cfs.getSSTables().size());

    // verify that the 1 indexed column was removed from the index
    assertEquals(1, index.deletes.size());
    assertEquals(index.deletes.get(0), index.inserts.get(0));
}
 

@Test
public void rangeTombstoneFilteringTest() throws Exception
{
    CompactionManager.instance.disableAutoCompaction();
    Keyspace keyspace = Keyspace.open(KSNAME);
    ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(CFNAME);

    // Inserting data
    String key = "k111";
    Mutation rm;
    ColumnFamily cf;

    rm = new Mutation(KSNAME, ByteBufferUtil.bytes(key));
    for (int i = 0; i < 40; i += 2)
        add(rm, i, 0);
    rm.apply();

    rm = new Mutation(KSNAME, ByteBufferUtil.bytes(key));
    cf = rm.addOrGet(CFNAME);
    delete(cf, 5, 10, 1);
    rm.apply();

    rm = new Mutation(KSNAME, ByteBufferUtil.bytes(key));
    cf = rm.addOrGet(CFNAME);
    delete(cf, 15, 20, 2);
    rm.apply();

    cf = cfs.getColumnFamily(QueryFilter.getSliceFilter(dk(key), CFNAME, b(11), b(14), false, Integer.MAX_VALUE, System.currentTimeMillis()));
    Collection<RangeTombstone> rt = rangeTombstones(cf);
    assertEquals(0, rt.size());

    cf = cfs.getColumnFamily(QueryFilter.getSliceFilter(dk(key), CFNAME, b(11), b(15), false, Integer.MAX_VALUE, System.currentTimeMillis()));
    rt = rangeTombstones(cf);
    assertEquals(1, rt.size());

    cf = cfs.getColumnFamily(QueryFilter.getSliceFilter(dk(key), CFNAME, b(20), b(25), false, Integer.MAX_VALUE, System.currentTimeMillis()));
    rt = rangeTombstones(cf);
    assertEquals(1, rt.size());

    cf = cfs.getColumnFamily(QueryFilter.getSliceFilter(dk(key), CFNAME, b(12), b(25), false, Integer.MAX_VALUE, System.currentTimeMillis()));
    rt = rangeTombstones(cf);
    assertEquals(1, rt.size());

    cf = cfs.getColumnFamily(QueryFilter.getSliceFilter(dk(key), CFNAME, b(25), b(35), false, Integer.MAX_VALUE, System.currentTimeMillis()));
    rt = rangeTombstones(cf);
    assertEquals(0, rt.size());

    cf = cfs.getColumnFamily(QueryFilter.getSliceFilter(dk(key), CFNAME, b(1), b(40), false, Integer.MAX_VALUE, System.currentTimeMillis()));
    rt = rangeTombstones(cf);
    assertEquals(2, rt.size());

    cf = cfs.getColumnFamily(QueryFilter.getSliceFilter(dk(key), CFNAME, b(7), b(17), false, Integer.MAX_VALUE, System.currentTimeMillis()));
    rt = rangeTombstones(cf);
    assertEquals(2, rt.size());

    cf = cfs.getColumnFamily(QueryFilter.getSliceFilter(dk(key), CFNAME, b(5), b(20), false, Integer.MAX_VALUE, System.currentTimeMillis()));
    rt = rangeTombstones(cf);
    assertEquals(2, rt.size());

    cf = cfs.getColumnFamily(QueryFilter.getSliceFilter(dk(key), CFNAME, b(5), b(15), false, Integer.MAX_VALUE, System.currentTimeMillis()));
    rt = rangeTombstones(cf);
    assertEquals(2, rt.size());

    cf = cfs.getColumnFamily(QueryFilter.getSliceFilter(dk(key), CFNAME, b(1), b(2), false, Integer.MAX_VALUE, System.currentTimeMillis()));
    rt = rangeTombstones(cf);
    assertEquals(0, rt.size());

    cf = cfs.getColumnFamily(QueryFilter.getSliceFilter(dk(key), CFNAME, b(1), b(5), false, Integer.MAX_VALUE, System.currentTimeMillis()));
    rt = rangeTombstones(cf);
    assertEquals(1, rt.size());

    cf = cfs.getColumnFamily(QueryFilter.getSliceFilter(dk(key), CFNAME, b(1), b(10), false, Integer.MAX_VALUE, System.currentTimeMillis()));
    rt = rangeTombstones(cf);
    assertEquals(1, rt.size());

    cf = cfs.getColumnFamily(QueryFilter.getSliceFilter(dk(key), CFNAME, b(5), b(6), false, Integer.MAX_VALUE, System.currentTimeMillis()));
    rt = rangeTombstones(cf);
    assertEquals(1, rt.size());

    cf = cfs.getColumnFamily(QueryFilter.getSliceFilter(dk(key), CFNAME, b(17), b(20), false, Integer.MAX_VALUE, System.currentTimeMillis()));
    rt = rangeTombstones(cf);
    assertEquals(1, rt.size());

    cf = cfs.getColumnFamily(QueryFilter.getSliceFilter(dk(key), CFNAME, b(17), b(18), false, Integer.MAX_VALUE, System.currentTimeMillis()));
    rt = rangeTombstones(cf);
    assertEquals(1, rt.size());

    ColumnSlice[] slices = new ColumnSlice[]{new ColumnSlice( b(1), b(10)), new ColumnSlice( b(16), b(20))};
    IDiskAtomFilter sqf = new SliceQueryFilter(slices, false, Integer.MAX_VALUE);
    cf = cfs.getColumnFamily( new QueryFilter(dk(key), CFNAME, sqf, System.currentTimeMillis()) );
    rt = rangeTombstones(cf);
    assertEquals(2, rt.size());
}
 

@Test
public void testRangeTombstoneCompaction() throws Exception
{
    Keyspace table = Keyspace.open(KSNAME);
    ColumnFamilyStore cfs = table.getColumnFamilyStore(CFNAME);
    ByteBuffer key = ByteBufferUtil.bytes("k4");

    // remove any existing sstables before starting
    cfs.truncateBlocking();
    cfs.disableAutoCompaction();
    cfs.setCompactionStrategyClass(SizeTieredCompactionStrategy.class.getCanonicalName());

    Mutation rm = new Mutation(KSNAME, key);
    for (int i = 0; i < 10; i += 2)
        add(rm, i, 0);
    rm.apply();
    cfs.forceBlockingFlush();

    rm = new Mutation(KSNAME, key);
    ColumnFamily cf = rm.addOrGet(CFNAME);
    for (int i = 0; i < 10; i += 2)
        delete(cf, 0, 7, 0);
    rm.apply();
    cfs.forceBlockingFlush();

    // there should be 2 sstables
    assertEquals(2, cfs.getSSTables().size());

    // compact down to single sstable
    CompactionManager.instance.performMaximal(cfs);
    assertEquals(1, cfs.getSSTables().size());

    // test the physical structure of the sstable i.e. rt & columns on disk
    SSTableReader sstable = cfs.getSSTables().iterator().next();
    OnDiskAtomIterator iter = sstable.getScanner().next();
    int cnt = 0;
    // after compaction, the first element should be an RT followed by the remaining non-deleted columns
    while(iter.hasNext())
    {
        OnDiskAtom atom = iter.next();
        if (cnt == 0)
            assertTrue(atom instanceof RangeTombstone);
        if (cnt > 0)
            assertTrue(atom instanceof Cell);
        cnt++;
    }
    assertEquals(2, cnt);
}
 

@Test
public void testScrubCorruptedCounterRow() throws IOException, WriteTimeoutException
{
    // skip the test when compression is enabled until CASSANDRA-9140 is complete
    assumeTrue(!Boolean.parseBoolean(System.getProperty("cassandra.test.compression", "false")));

    CompactionManager.instance.disableAutoCompaction();
    Keyspace keyspace = Keyspace.open(KEYSPACE);
    ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(COUNTER_CF);
    cfs.clearUnsafe();

    fillCounterCF(cfs, 2);

    List<Row> rows = cfs.getRangeSlice(Util.range("", ""), null, new IdentityQueryFilter(), 1000);
    assertEquals(2, rows.size());

    SSTableReader sstable = cfs.getSSTables().iterator().next();

    // overwrite one row with garbage
    long row0Start = sstable.getPosition(RowPosition.ForKey.get(ByteBufferUtil.bytes("0"), sstable.partitioner), SSTableReader.Operator.EQ).position;
    long row1Start = sstable.getPosition(RowPosition.ForKey.get(ByteBufferUtil.bytes("1"), sstable.partitioner), SSTableReader.Operator.EQ).position;
    long startPosition = row0Start < row1Start ? row0Start : row1Start;
    long endPosition = row0Start < row1Start ? row1Start : row0Start;

    RandomAccessFile file = new RandomAccessFile(sstable.getFilename(), "rw");
    file.seek(startPosition);
    file.writeBytes(StringUtils.repeat('z', (int) (endPosition - startPosition)));
    file.close();

    // with skipCorrupted == false, the scrub is expected to fail
    Scrubber scrubber = new Scrubber(cfs, sstable, false, false);
    try
    {
        scrubber.scrub();
        fail("Expected a CorruptSSTableException to be thrown");
    }
    catch (IOError err) {}

    // with skipCorrupted == true, the corrupt row will be skipped
    scrubber = new Scrubber(cfs, sstable, true, false);
    scrubber.scrub();
    scrubber.close();
    assertEquals(1, cfs.getSSTables().size());

    // verify that we can read all of the rows, and there is now one less row
    rows = cfs.getRangeSlice(Util.range("", ""), null, new IdentityQueryFilter(), 1000);
    assertEquals(1, rows.size());
}
 

@Override
public int forceKeyspaceCleanup(final String keyspaceName, final String... columnFamilies) {
    return CompactionManager.AllSSTableOpStatus.SUCCESSFUL.statusCode;
}
 

@Test
public void testKeyCache() throws ExecutionException, InterruptedException
{
    CompactionManager.instance.disableAutoCompaction();

    Keyspace keyspace = Keyspace.open(KEYSPACE1);
    ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(COLUMN_FAMILY1);

    // just to make sure that everything is clean
    CacheService.instance.invalidateKeyCache();

    // KeyCache should start at size 0 if we're caching X% of zero data.
    assertKeyCacheSize(0, KEYSPACE1, COLUMN_FAMILY1);

    DecoratedKey key1 = Util.dk("key1");
    DecoratedKey key2 = Util.dk("key2");
    Mutation rm;

    // inserts
    rm = new Mutation(KEYSPACE1, key1.getKey());
    rm.add(COLUMN_FAMILY1, Util.cellname("1"), ByteBufferUtil.EMPTY_BYTE_BUFFER, 0);
    rm.apply();
    rm = new Mutation(KEYSPACE1, key2.getKey());
    rm.add(COLUMN_FAMILY1, Util.cellname("2"), ByteBufferUtil.EMPTY_BYTE_BUFFER, 0);
    rm.apply();

    // to make sure we have SSTable
    cfs.forceBlockingFlush();

    // reads to cache key position
    cfs.getColumnFamily(QueryFilter.getSliceFilter(key1,
                                                   COLUMN_FAMILY1,
                                                   Composites.EMPTY,
                                                   Composites.EMPTY,
                                                   false,
                                                   10,
                                                   System.currentTimeMillis()));

    cfs.getColumnFamily(QueryFilter.getSliceFilter(key2,
                                                   COLUMN_FAMILY1,
                                                   Composites.EMPTY,
                                                   Composites.EMPTY,
                                                   false,
                                                   10,
                                                   System.currentTimeMillis()));

    assertKeyCacheSize(2, KEYSPACE1, COLUMN_FAMILY1);

    Set<SSTableReader> readers = cfs.getDataTracker().getSSTables();
    Refs<SSTableReader> refs = Refs.tryRef(readers);
    if (refs == null)
        throw new IllegalStateException();

    Util.compactAll(cfs, Integer.MAX_VALUE).get();
    // after compaction cache should have entries for new SSTables,
    // but since we have kept a reference to the old sstables,
    // if we had 2 keys in cache previously it should become 4
    assertKeyCacheSize(4, KEYSPACE1, COLUMN_FAMILY1);

    refs.release();

    Uninterruptibles.sleepUninterruptibly(10, TimeUnit.MILLISECONDS);;
    while (ScheduledExecutors.nonPeriodicTasks.getActiveCount() + ScheduledExecutors.nonPeriodicTasks.getQueue().size() > 0);

    // after releasing the reference this should drop to 2
    assertKeyCacheSize(2, KEYSPACE1, COLUMN_FAMILY1);

    // re-read same keys to verify that key cache didn't grow further
    cfs.getColumnFamily(QueryFilter.getSliceFilter(key1,
                                                   COLUMN_FAMILY1,
                                                   Composites.EMPTY,
                                                   Composites.EMPTY,
                                                   false,
                                                   10,
                                                   System.currentTimeMillis()));

    cfs.getColumnFamily(QueryFilter.getSliceFilter(key2,
                                                   COLUMN_FAMILY1,
                                                   Composites.EMPTY,
                                                   Composites.EMPTY,
                                                   false,
                                                   10,
                                                   System.currentTimeMillis()));

    assertKeyCacheSize(2, KEYSPACE1, COLUMN_FAMILY1);
}
 

@Test
public void testKeyCacheLoad() throws Exception
{
    CompactionManager.instance.disableAutoCompaction();

    ColumnFamilyStore store = Keyspace.open(KEYSPACE1).getColumnFamilyStore(COLUMN_FAMILY2);

    // empty the cache
    CacheService.instance.invalidateKeyCache();
    assertKeyCacheSize(0, KEYSPACE1, COLUMN_FAMILY2);

    // insert data and force to disk
    insertData(KEYSPACE1, COLUMN_FAMILY2, 0, 100);
    store.forceBlockingFlush();

    // populate the cache
    readData(KEYSPACE1, COLUMN_FAMILY2, 0, 100);
    assertKeyCacheSize(100, KEYSPACE1, COLUMN_FAMILY2);

    // really? our caches don't implement the map interface? (hence no .addAll)
    Map<KeyCacheKey, RowIndexEntry> savedMap = new HashMap<KeyCacheKey, RowIndexEntry>();
    for (KeyCacheKey k : CacheService.instance.keyCache.getKeySet())
    {
        if (k.desc.ksname.equals(KEYSPACE1) && k.desc.cfname.equals(COLUMN_FAMILY2))
            savedMap.put(k, CacheService.instance.keyCache.get(k));
    }

    // force the cache to disk
    CacheService.instance.keyCache.submitWrite(Integer.MAX_VALUE).get();

    CacheService.instance.invalidateKeyCache();
    assertKeyCacheSize(0, KEYSPACE1, COLUMN_FAMILY2);

    CacheService.instance.keyCache.loadSaved(store);
    assertKeyCacheSize(savedMap.size(), KEYSPACE1, COLUMN_FAMILY2);

    // probably it's better to add equals/hashCode to RowIndexEntry...
    for (Map.Entry<KeyCacheKey, RowIndexEntry> entry : savedMap.entrySet())
    {
        RowIndexEntry expected = entry.getValue();
        RowIndexEntry actual = CacheService.instance.keyCache.get(entry.getKey());
        assertEquals(expected.position, actual.position);
        assertEquals(expected.columnsIndex(), actual.columnsIndex());
        if (expected.isIndexed())
        {
            assertEquals(expected.deletionTime(), actual.deletionTime());
        }
    }
}
 

private void track(Set<ColumnDefinition> columns)
{
    // if SI hasn't been initialized that means that this instance
    // was created for validation purposes only, so we don't have do anything here.
    // And only reason baseCfs would be null is if coming through the CFMetaData.validate() path, which only
    // checks that the SI 'looks' legit, but then throws away any created instances - fml, this 2I api sux
    if (!isInitialized || baseCfs == null)
        return;

    if (keyValidator == null)
        keyValidator = baseCfs.metadata.getKeyValidator();

    Set<SSTableReader> sstables = baseCfs.getDataTracker().getSSTables();
    NavigableMap<SSTableReader, Map<ByteBuffer, ColumnIndex>> toRebuild = new TreeMap<>(new Comparator<SSTableReader>()
    {
        @Override
        public int compare(SSTableReader a, SSTableReader b)
        {
            return Integer.compare(a.descriptor.generation, b.descriptor.generation);
        }
    });

    for (ColumnDefinition column : columns)
    {
        ColumnIndex index = indexedColumns.get(column.name);
        if (index == null)
        {
            ColumnIndex newIndex = new ColumnIndex(keyValidator, column, getComparator(column));
            index = indexedColumns.putIfAbsent(column.name, newIndex);

            if (index != null)
                continue;

            // on restart, sstables are loaded into DataTracker before 2I are hooked up (and init() invoked),
            // so we need to explicitly load sstables per index.
            for (SSTableReader sstable : newIndex.init(sstables))
            {
                Map<ByteBuffer, ColumnIndex> perSSTable = toRebuild.get(sstable);
                if (perSSTable == null)
                    toRebuild.put(sstable, (perSSTable = new HashMap<>()));

                perSSTable.put(column.name, newIndex);
            }
        }
    }

    // schedule rebuild of the missing indexes. Property affects both existing and newly (re-)created
    // indexes since SecondaryIndex API makes no distinction between them.
    if (Boolean.parseBoolean(System.getProperty("cassandra.sasi.rebuild_on_start", "true")))
        CompactionManager.instance.submitIndexBuild(new IndexBuilder(toRebuild));
}
 

public Future<?> submitWrite(int keysToSave)
{
    return CompactionManager.instance.submitCacheWrite(getWriter(keysToSave));
}
 

private static void dropKeyspace(String ksName)
{
    KSMetaData ksm = Schema.instance.getKSMetaData(ksName);
    String snapshotName = Keyspace.getTimestampedSnapshotName(ksName);

    CompactionManager.instance.interruptCompactionFor(ksm.cfMetaData().values(), true);

    Keyspace keyspace = Keyspace.open(ksm.name);

    // remove all cfs from the keyspace instance.
    List<UUID> droppedCfs = new ArrayList<>();
    for (CFMetaData cfm : ksm.cfMetaData().values())
    {
        ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(cfm.cfName);

        Schema.instance.purge(cfm);

        if (!StorageService.instance.isClientMode())
        {
            if (DatabaseDescriptor.isAutoSnapshot())
                cfs.snapshot(snapshotName);
            Keyspace.open(ksm.name).dropCf(cfm.cfId);
        }

        droppedCfs.add(cfm.cfId);
    }

    // remove the keyspace from the static instances.
    Keyspace.clear(ksm.name);
    Schema.instance.clearKeyspaceDefinition(ksm);

    keyspace.writeOrder.awaitNewBarrier();

    // force a new segment in the CL
    CommitLog.instance.forceRecycleAllSegments(droppedCfs);

    if (!StorageService.instance.isClientMode())
    {
        MigrationManager.instance.notifyDropKeyspace(ksm);
    }
}
 

private Relationship compare(ContextState leftState, ContextState rightState)
{
    long leftClock = leftState.getClock();
    long leftCount = leftState.getCount();
    long rightClock = rightState.getClock();
    long rightCount = rightState.getCount();

    if (leftState.isGlobal() || rightState.isGlobal())
    {
        if (leftState.isGlobal() && rightState.isGlobal())
        {
            if (leftClock == rightClock)
            {
                // Can happen if an sstable gets lost and disk failure policy is set to 'best effort'
                if (leftCount != rightCount && CompactionManager.isCompactionManager.get())
                {
                    logger.warn("invalid global counter shard detected; ({}, {}, {}) and ({}, {}, {}) differ only in "
                                + "count; will pick highest to self-heal on compaction",
                                leftState.getCounterId(), leftClock, leftCount,
                                rightState.getCounterId(), rightClock, rightCount);
                }

                if (leftCount > rightCount)
                    return Relationship.GREATER_THAN;
                else if (leftCount == rightCount)
                    return Relationship.EQUAL;
                else
                    return Relationship.LESS_THAN;
            }
            else
            {
                return leftClock > rightClock ? Relationship.GREATER_THAN : Relationship.LESS_THAN;
            }
        }
        else // only one is global - keep that one
        {
            return leftState.isGlobal() ? Relationship.GREATER_THAN : Relationship.LESS_THAN;
        }
    }

    if (leftState.isLocal() || rightState.isLocal())
    {
        // Local id and at least one is a local shard.
        if (leftState.isLocal() && rightState.isLocal())
            return Relationship.DISJOINT;
        else // only one is local - keep that one
            return leftState.isLocal() ? Relationship.GREATER_THAN : Relationship.LESS_THAN;
    }

    // both are remote shards
    if (leftClock == rightClock)
    {
        // We should never see non-local shards w/ same id+clock but different counts. However, if we do
        // we should "heal" the problem by being deterministic in our selection of shard - and
        // log the occurrence so that the operator will know something is wrong.
        if (leftCount != rightCount && CompactionManager.isCompactionManager.get())
        {
            logger.warn("invalid remote counter shard detected; ({}, {}, {}) and ({}, {}, {}) differ only in "
                        + "count; will pick highest to self-heal on compaction",
                        leftState.getCounterId(), leftClock, leftCount,
                        rightState.getCounterId(), rightClock, rightCount);
        }

        if (leftCount > rightCount)
            return Relationship.GREATER_THAN;
        else if (leftCount == rightCount)
            return Relationship.EQUAL;
        else
            return Relationship.LESS_THAN;
    }
    else
    {
        if ((leftClock >= 0 && rightClock > 0 && leftClock >= rightClock)
                || (leftClock < 0 && (rightClock > 0 || leftClock < rightClock)))
            return Relationship.GREATER_THAN;
        else
            return Relationship.LESS_THAN;
    }
}
 

@Override
public void buildIndexes(Collection<SSTableReader> sstablesToRebuild, Set<String> indexNames)
{
    NavigableMap<SSTableReader, Map<ByteBuffer, ColumnIndex>> sstables = new TreeMap<>(new Comparator<SSTableReader>()
    {
        @Override
        public int compare(SSTableReader a, SSTableReader b)
        {
            return Integer.compare(a.descriptor.generation, b.descriptor.generation);
        }
    });

    Map<ByteBuffer, ColumnIndex> indexes = new HashMap<>();
    for (ColumnIndex index : indexedColumns.values())
    {
        Iterator<String> iterator = indexNames.iterator();

        while (iterator.hasNext())
        {
            String indexName = iterator.next();
            if (index.getIndexName().equals(indexName))
            {
                index.dropData(FBUtilities.timestampMicros());

                ColumnDefinition indexToBuild = index.getDefinition();
                indexes.put(indexToBuild.name, index);
                iterator.remove();
                break;
            }
        }
    }

    if (indexes.isEmpty())
        return;

    for (SSTableReader sstable : sstablesToRebuild)
        sstables.put(sstable, indexes);

    try
    {
        FBUtilities.waitOnFuture(CompactionManager.instance.submitIndexBuild(new IndexBuilder(sstables)));
    }
    catch (Exception e)
    {
        logger.error("Failed index build task", e);
    }
}