org.apache.cassandra.io.sstable.SSTableReader Java Examples

/**
 * removes all sstables that are not busy compacting.
 */
public void unreferenceSSTables()
{
    Set<SSTableReader> notCompacting;

    View currentView, newView;
    do
    {
        currentView = view.get();
        notCompacting = currentView.nonCompactingSStables();
        newView = currentView.replace(notCompacting, Collections.<SSTableReader>emptySet());
    }
    while (!view.compareAndSet(currentView, newView));

    if (notCompacting.isEmpty())
    {
        // notifySSTablesChanged -> LeveledManifest.promote doesn't like a no-op "promotion"
        return;
    }
    notifySSTablesChanged(notCompacting, Collections.<SSTableReader>emptySet(), OperationType.UNKNOWN);
    removeOldSSTablesSize(notCompacting);
    releaseReferences(notCompacting, true);
}
 

/**
 * Returns a list of KeyScanners given sstables and a range on which to scan.
 * The default implementation simply grab one SSTableScanner per-sstable, but overriding this method
 * allow for a more memory efficient solution if we know the sstable don't overlap (see
 * LeveledCompactionStrategy for instance).
 */
public ScannerList getScanners(Collection<SSTableReader> sstables, Range<Token> range)
{
    RateLimiter limiter = CompactionManager.instance.getRateLimiter();
    ArrayList<ISSTableScanner> scanners = new ArrayList<ISSTableScanner>();
    try
    {
        for (SSTableReader sstable : sstables)
            scanners.add(sstable.getScanner(range, limiter));
    }
    catch (Throwable t)
    {
        try
        {
            new ScannerList(scanners).close();
        }
        catch (Throwable t2)
        {
            t.addSuppressed(t2);
        }
        throw t;
    }
    return new ScannerList(scanners);
}
 

public OutgoingFileMessage(SSTableReader sstable, Ref ref, int sequenceNumber, long estimatedKeys, List<Pair<Long, Long>> sections, long repairedAt)
{
    super(Type.FILE);
    this.sstable = sstable;
    this.ref = ref;

    CompressionInfo compressionInfo = null;
    if (sstable.compression)
    {
        CompressionMetadata meta = sstable.getCompressionMetadata();
        compressionInfo = new CompressionInfo(meta.getChunksForSections(sections), meta.parameters);
    }
    this.header = new FileMessageHeader(sstable.metadata.cfId,
                                        sequenceNumber,
                                        sstable.descriptor.version.toString(),
                                        estimatedKeys,
                                        sections,
                                        compressionInfo,
                                        repairedAt);
}
 

/**
 * Convenience method to compute the replay position for a group of SSTables.
 * @param sstables
 * @return the most recent (highest) replay position
 */
public static ReplayPosition getReplayPosition(Iterable<? extends SSTableReader> sstables)
{
    if (Iterables.isEmpty(sstables))
        return NONE;

    Function<SSTableReader, ReplayPosition> f = new Function<SSTableReader, ReplayPosition>()
    {
        public ReplayPosition apply(SSTableReader sstable)
        {
            return sstable.getReplayPosition();
        }
    };
    Ordering<ReplayPosition> ordering = Ordering.from(ReplayPosition.comparator);
    return ordering.max(Iterables.transform(sstables, f));
}
 

@Override
public synchronized void startup()
{
    super.startup();
    for (SSTableReader sstable : cfs.getSSTables())
    {
        if (sstable.openReason != SSTableReader.OpenReason.EARLY)
        {
            if (sstable.isRepaired())
                repaired.addSSTable(sstable);
            else
                unrepaired.addSSTable(sstable);
        }
    }
    repaired.startup();
    unrepaired.startup();
}
 

@Override
public synchronized ScannerList getScanners(Collection<SSTableReader> sstables, Range<Token> range)
{
    List<SSTableReader> repairedSSTables = new ArrayList<>();
    List<SSTableReader> unrepairedSSTables = new ArrayList<>();
    for (SSTableReader sstable : sstables)
        if (sstable.isRepaired())
            repairedSSTables.add(sstable);
        else
            unrepairedSSTables.add(sstable);
    ScannerList repairedScanners = repaired.getScanners(repairedSSTables, range);
    ScannerList unrepairedScanners = unrepaired.getScanners(unrepairedSSTables, range);
    List<ISSTableScanner> scanners = new ArrayList<>(repairedScanners.scanners.size() + unrepairedScanners.scanners.size());
    scanners.addAll(repairedScanners.scanners);
    scanners.addAll(unrepairedScanners.scanners);
    return new ScannerList(scanners);
}
 

@Override
public synchronized AbstractCompactionTask getNextBackgroundTask(int gcBefore)
{
    if (!isEnabled())
        return null;

    while (true)
    {
        List<SSTableReader> latestBucket = getNextBackgroundSSTables(gcBefore);

        if (latestBucket.isEmpty())
            return null;

        if (cfs.getDataTracker().markCompacting(latestBucket))
            return new CompactionTask(cfs, latestBucket, gcBefore, false);
    }
}
 

private List<SSTableReader> getCompactionCandidates(Iterable<SSTableReader> candidateSSTables, long now, int base)
{
    Iterable<SSTableReader> candidates = filterOldSSTables(Lists.newArrayList(candidateSSTables), options.maxSSTableAge, now);

    List<List<SSTableReader>> buckets = getBuckets(createSSTableAndMinTimestampPairs(candidates), options.baseTime, base, now);
    logger.debug("Compaction buckets are {}", buckets);
    updateEstimatedCompactionsByTasks(buckets);
    List<SSTableReader> mostInteresting = newestBucket(buckets,
                                                       cfs.getMinimumCompactionThreshold(),
                                                       cfs.getMaximumCompactionThreshold(),
                                                       now,
                                                       options.baseTime);
    if (!mostInteresting.isEmpty())
        return mostInteresting;
    return null;
}
 

public synchronized AbstractCompactionTask getNextBackgroundTask(int gcBefore)
{
    if (!isEnabled())
        return null;

    while (true)
    {
        List<SSTableReader> hottestBucket = getNextBackgroundSSTables(gcBefore);

        if (hottestBucket.isEmpty())
            return null;

        if (cfs.getDataTracker().markCompacting(hottestBucket))
            return new CompactionTask(cfs, hottestBucket, gcBefore, false);
    }
}
 

/**
 * Removes all sstables with max timestamp older than maxSSTableAge.
 * @param sstables all sstables to consider
 * @param maxSSTableAge the age in milliseconds when an SSTable stops participating in compactions
 * @param now current time. SSTables with max timestamp less than (now - maxSSTableAge) are filtered.
 * @return a list of sstables with the oldest sstables excluded
 */
@VisibleForTesting
static Iterable<SSTableReader> filterOldSSTables(List<SSTableReader> sstables, long maxSSTableAge, long now)
{
    if (maxSSTableAge == 0)
        return sstables;
    final long cutoff = now - maxSSTableAge;
    return Iterables.filter(sstables, new Predicate<SSTableReader>()
    {
        @Override
        public boolean apply(SSTableReader sstable)
        {
            return sstable.getMaxTimestamp() >= cutoff;
        }
    });
}
 

@Test
public void testTrackTimesRangeTombstoneWithData() throws ExecutionException, InterruptedException
{
    Keyspace ks = Keyspace.open(KSNAME);
    ColumnFamilyStore cfs = ks.getColumnFamilyStore(CFNAME);
    cfs.truncateBlocking();
    String key = "rt_times";
    Mutation rm = new Mutation(KSNAME, ByteBufferUtil.bytes(key));
    add(rm, 5, 999);
    rm.apply();
    key = "rt_times2";
    rm = new Mutation(KSNAME, ByteBufferUtil.bytes(key));
    ColumnFamily cf = rm.addOrGet(CFNAME);
    int timestamp = (int)(System.currentTimeMillis()/1000);
    cf.delete(new DeletionInfo(b(1), b(2), cfs.getComparator(), 1000, timestamp));
    rm.apply();
    cfs.forceBlockingFlush();
    SSTableReader sstable = cfs.getSSTables().iterator().next();
    assertTimes(sstable.getSSTableMetadata(), 999, 1000, Integer.MAX_VALUE);
    cfs.forceMajorCompaction();
    sstable = cfs.getSSTables().iterator().next();
    assertTimes(sstable.getSSTableMetadata(), 999, 1000, Integer.MAX_VALUE);
}
 

@Test
public void testTrackTimesRowTombstone() throws ExecutionException, InterruptedException
{
    Keyspace ks = Keyspace.open(KSNAME);
    ColumnFamilyStore cfs = ks.getColumnFamilyStore(CFNAME);
    cfs.truncateBlocking();
    String key = "rt_times";
    Mutation rm = new Mutation(KSNAME, ByteBufferUtil.bytes(key));
    ColumnFamily cf = rm.addOrGet(CFNAME);
    long timestamp = System.currentTimeMillis();
    cf.delete(new DeletionInfo(1000, (int)(timestamp/1000)));
    rm.apply();
    cfs.forceBlockingFlush();
    SSTableReader sstable = cfs.getSSTables().iterator().next();
    assertTimes(sstable.getSSTableMetadata(), 1000, 1000, (int)(timestamp/1000));
    cfs.forceMajorCompaction();
    sstable = cfs.getSSTables().iterator().next();
    assertTimes(sstable.getSSTableMetadata(), 1000, 1000, (int)(timestamp/1000));
}
 

/**
 * @param buckets list of buckets from which to return the most interesting, where "interesting" is the total hotness for reads
 * @param minThreshold minimum number of sstables in a bucket to qualify as interesting
 * @param maxThreshold maximum number of sstables to compact at once (the returned bucket will be trimmed down to this)
 * @return a bucket (list) of sstables to compact
 */
public static List<SSTableReader> mostInterestingBucket(List<List<SSTableReader>> buckets, int minThreshold, int maxThreshold)
{
    // skip buckets containing less than minThreshold sstables, and limit other buckets to maxThreshold sstables
    final List<Pair<List<SSTableReader>, Double>> prunedBucketsAndHotness = new ArrayList<>(buckets.size());
    for (List<SSTableReader> bucket : buckets)
    {
        Pair<List<SSTableReader>, Double> bucketAndHotness = trimToThresholdWithHotness(bucket, maxThreshold);
        if (bucketAndHotness != null && bucketAndHotness.left.size() >= minThreshold)
            prunedBucketsAndHotness.add(bucketAndHotness);
    }
    if (prunedBucketsAndHotness.isEmpty())
        return Collections.emptyList();

    Pair<List<SSTableReader>, Double> hottest = Collections.max(prunedBucketsAndHotness, bucketsByHotnessComparator);
    return hottest.left;
}
 

@Test
/* 
 *  The schema is 
 *      CREATE TABLE cql_keyspace.table1 (k int PRIMARY KEY, v1 text, v2 int)
 * */
public void shouldImportCqlTable() throws IOException, URISyntaxException
{
    String cql_keyspace = "cql_keyspace";
    String cql_table = "table1";
    String jsonUrl = resourcePath("CQLTable.json");
    File tempSS = tempSSTableFile(cql_keyspace, cql_table);
    new SSTableImport(true).importJson(jsonUrl, cql_keyspace, cql_table, tempSS.getPath());
    SSTableReader reader = SSTableReader.open(Descriptor.fromFilename(tempSS.getPath()));
    Keyspace.open(cql_keyspace).getColumnFamilyStore(cql_table).addSSTable(reader);
    
    UntypedResultSet result = QueryProcessor.executeOnceInternal(String.format("SELECT * FROM %s.%s", cql_keyspace, cql_table));
    assertThat(result.size(), is(2));
    assertThat(result, hasItem(withElements(1, "NY", 1980)));
    assertThat(result, hasItem(withElements(2, "CA", 2014)));
    reader.selfRef().release();
}
 

@Test
public void testTrackTimesRangeTombstone() throws ExecutionException, InterruptedException
{
    Keyspace ks = Keyspace.open(KSNAME);
    ColumnFamilyStore cfs = ks.getColumnFamilyStore(CFNAME);
    cfs.truncateBlocking();
    String key = "rt_times";
    Mutation rm = new Mutation(KSNAME, ByteBufferUtil.bytes(key));
    ColumnFamily cf = rm.addOrGet(CFNAME);
    long timestamp = System.currentTimeMillis();
    cf.delete(new DeletionInfo(b(1), b(2), cfs.getComparator(), 1000, (int)(timestamp/1000)));
    rm.apply();
    cfs.forceBlockingFlush();
    SSTableReader sstable = cfs.getSSTables().iterator().next();
    assertTimes(sstable.getSSTableMetadata(), 1000, 1000, (int)(timestamp/1000));
    cfs.forceMajorCompaction();
    sstable = cfs.getSSTables().iterator().next();
    assertTimes(sstable.getSSTableMetadata(), 1000, 1000, (int)(timestamp/1000));
}
 

/**
 * @param buckets list of buckets, sorted from newest to oldest, from which to return the newest bucket within thresholds.
 * @param minThreshold minimum number of sstables in a bucket to qualify.
 * @param maxThreshold maximum number of sstables to compact at once (the returned bucket will be trimmed down to this).
 * @return a bucket (list) of sstables to compact.
 */
@VisibleForTesting
static List<SSTableReader> newestBucket(List<List<SSTableReader>> buckets, int minThreshold, int maxThreshold, long now, long baseTime)
{
    // If the "incoming window" has at least minThreshold SSTables, choose that one.
    // For any other bucket, at least 2 SSTables is enough.
    // In any case, limit to maxThreshold SSTables.
    Target incomingWindow = getInitialTarget(now, baseTime);
    for (List<SSTableReader> bucket : buckets)
    {
        if (bucket.size() >= minThreshold ||
                (bucket.size() >= 2 && !incomingWindow.onTarget(bucket.get(0).getMinTimestamp())))
            return trimToThreshold(bucket, maxThreshold);
    }
    return Collections.emptyList();
}
 

public static long getMaxDataAge(Collection<SSTableReader> sstables)
{
    long max = 0;
    for (SSTableReader sstable : sstables)
    {
        if (sstable.maxDataAge > max)
            max = sstable.maxDataAge;
    }
    return max;
}
 

@Test
public void testSizeMetric()
{
    Keyspace keyspace = Keyspace.open("Keyspace1");
    ColumnFamilyStore store = keyspace.getColumnFamilyStore("Standard1");
    store.disableAutoCompaction();

    store.truncateBlocking();

    assertEquals(0, store.metric.liveDiskSpaceUsed.count());
    assertEquals(0, store.metric.totalDiskSpaceUsed.count());

    for (int j = 0; j < 10; j++)
    {
        ByteBuffer key = ByteBufferUtil.bytes(String.valueOf(j));
        Mutation rm = new Mutation("Keyspace1", key);
        rm.add("Standard1", cellname("0"), ByteBufferUtil.EMPTY_BYTE_BUFFER, j);
        rm.apply();
    }
    store.forceBlockingFlush();
    Collection<SSTableReader> sstables = store.getSSTables();
    long size = 0;
    for (SSTableReader reader : sstables)
    {
        size += reader.bytesOnDisk();
    }

    // size metrics should show the sum of all SSTable sizes
    assertEquals(size, store.metric.liveDiskSpaceUsed.count());
    assertEquals(size, store.metric.totalDiskSpaceUsed.count());

    store.truncateBlocking();

    // after truncate, size metrics should be down to 0
    assertEquals(0, store.metric.liveDiskSpaceUsed.count());
    assertEquals(0, store.metric.totalDiskSpaceUsed.count());

    store.enableAutoCompaction();
}
 

void maybeRefreshOverlaps()
{
    for (SSTableReader reader : overlappingSSTables)
    {
        if (reader.isMarkedCompacted())
        {
            refreshOverlaps();
            return;
        }
    }
}
 

private static Map<SSTableReader, Double> getHotnessMap(Collection<SSTableReader> sstables)
{
    Map<SSTableReader, Double> hotness = new HashMap<>();
    for (SSTableReader sstable : sstables)
        hotness.put(sstable, hotness(sstable));
    return hotness;
}
 

private void refreshOverlaps()
{
    if (this.overlappingSSTables != null)
        overlappingSSTables.release();

    if (compacting == null)
        overlappingSSTables = Refs.tryRef(Collections.<SSTableReader>emptyList());
    else
        overlappingSSTables = cfs.getAndReferenceOverlappingSSTables(compacting);
    this.overlappingTree = DataTracker.buildIntervalTree(overlappingSSTables);
}
 

public Collection<AbstractCompactionTask> getMaximalTask(final int gcBefore)
{
    Iterable<SSTableReader> filteredSSTables = filterSuspectSSTables(sstables);
    if (Iterables.isEmpty(filteredSSTables))
        return null;
    if (!cfs.getDataTracker().markCompacting(ImmutableList.copyOf(filteredSSTables)))
        return null;
    return Arrays.<AbstractCompactionTask>asList(new CompactionTask(cfs, filteredSSTables, gcBefore, false));
}
 

public double getDroppableTombstoneRatio()
{
    double allDroppable = 0;
    long allColumns = 0;
    int localTime = (int)(System.currentTimeMillis()/1000);

    for (SSTableReader sstable : getSSTables())
    {
        allDroppable += sstable.getDroppableTombstonesBefore(localTime - sstable.metadata.getGcGraceSeconds());
        allColumns += sstable.getEstimatedColumnCount().mean() * sstable.getEstimatedColumnCount().count();
    }
    return allColumns > 0 ? allDroppable / allColumns : 0;
}
 

private void releaseReferences(Iterable<SSTableReader> oldSSTables, boolean tolerateCompacted)
{
    for (SSTableReader sstable : oldSSTables)
    {
        boolean firstToCompact = sstable.markObsolete();
        assert tolerateCompacted || firstToCompact : sstable + " was already marked compacted";
        sstable.selfRef().release();
    }
}
 

/**
 *
 * @param oldSSTables
 * @param allReplacements
 * @param compactionType
 */
// note that this DOES NOT insert the replacement sstables, it only removes the old sstables and notifies any listeners
// that they have been replaced by the provided sstables, which must have been performed by an earlier replaceReaders() call
public void markCompactedSSTablesReplaced(Collection<SSTableReader> oldSSTables, Collection<SSTableReader> allReplacements, OperationType compactionType)
{
    removeSSTablesFromTracker(oldSSTables);
    releaseReferences(oldSSTables, false);
    notifySSTablesChanged(oldSSTables, allReplacements, compactionType);
    addNewSSTablesSize(allReplacements);
}
 

private List<SSTableReader> getNextBackgroundSSTables(final int gcBefore)
{
    if (!isEnabled())
        return Collections.emptyList();

    // make local copies so they can't be changed out from under us mid-method
    int minThreshold = cfs.getMinimumCompactionThreshold();
    int maxThreshold = cfs.getMaximumCompactionThreshold();

    Iterable<SSTableReader> candidates = filterSuspectSSTables(Sets.intersection(cfs.getUncompactingSSTables(), sstables));

    List<List<SSTableReader>> buckets = getBuckets(createSSTableAndLengthPairs(candidates), options.bucketHigh, options.bucketLow, options.minSSTableSize);
    logger.debug("Compaction buckets are {}", buckets);
    updateEstimatedCompactionsByTasks(buckets);
    List<SSTableReader> mostInteresting = mostInterestingBucket(buckets, minThreshold, maxThreshold);
    if (!mostInteresting.isEmpty())
        return mostInteresting;

    // if there is no sstable to compact in standard way, try compacting single sstable whose droppable tombstone
    // ratio is greater than threshold.
    List<SSTableReader> sstablesWithTombstones = new ArrayList<>();
    for (SSTableReader sstable : candidates)
    {
        if (worthDroppingTombstones(sstable, gcBefore))
            sstablesWithTombstones.add(sstable);
    }
    if (sstablesWithTombstones.isEmpty())
        return Collections.emptyList();

    Collections.sort(sstablesWithTombstones, new SSTableReader.SizeComparator());
    return Collections.singletonList(sstablesWithTombstones.get(0));
}
 

@Override
public synchronized AbstractCompactionTask getUserDefinedTask(Collection<SSTableReader> sstables, int gcBefore)
{
    assert !sstables.isEmpty(); // checked for by CM.submitUserDefined

    if (!cfs.getDataTracker().markCompacting(sstables))
    {
        logger.debug("Unable to mark {} for compaction; probably a background compaction got to it first.  You can disable background compactions temporarily if this is a problem", sstables);
        return null;
    }

    return new CompactionTask(cfs, sstables, gcBefore, false).setUserDefined(true);
}
 

/** (Re)initializes the tracker, purging all references. */
void init()
{
    view.set(new View(
                     ImmutableList.of(new Memtable(cfstore)),
                     ImmutableList.<Memtable>of(),
                     Collections.<SSTableReader, SSTableReader>emptyMap(),
                     Collections.<SSTableReader>emptySet(),
                     Collections.<SSTableReader>emptySet(),
                     SSTableIntervalTree.empty()));
}
 

private void updateEstimatedCompactionsByTasks(List<List<SSTableReader>> tasks)
{
    int n = 0;
    for (List<SSTableReader> bucket : tasks)
    {
        if (bucket.size() >= cfs.getMinimumCompactionThreshold())
            n += Math.ceil((double)bucket.size() / cfs.getMaximumCompactionThreshold());
    }
    estimatedRemainingTasks = n;
}
 

public List<SSTableReader> sstablesInBounds(AbstractBounds<RowPosition> rowBounds)
{
    if (intervalTree.isEmpty())
        return Collections.emptyList();
    RowPosition stopInTree = rowBounds.right.isMinimum(liveMemtables.get(0).cfs.partitioner) ? intervalTree.max() : rowBounds.right;
    return intervalTree.search(Interval.<RowPosition, SSTableReader>create(rowBounds.left, stopInTree));
}