com.netflix.astyanax.model.ConsistencyLevel Java Examples

The following examples show how to use com.netflix.astyanax.model.ConsistencyLevel. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: AstyanaxQueueDAO.java    From emodb with Apache License 2.0 6 votes vote down vote up
@Override
public Map<UUID, ByteBuffer> findMaxRecords(Collection<UUID> dataIds) {
    // Finding the max using a reversed column range shouldn't have to worry about skipping tombstones since
    // we always delete smaller column values before deleting larger column values--scanning will hit the max
    // before needing to skip over tombstones.
    Map<UUID, ByteBuffer> resultMap = Maps.newHashMap();
    for (List<UUID> batch : Iterables.partition(dataIds, 10)) {
        Rows<UUID, ByteBuffer> rows = execute(
                _keyspace.prepareQuery(CF_DEDUP_DATA, ConsistencyLevel.CL_LOCAL_QUORUM)
                        .getKeySlice(batch)
                        .withColumnRange(new RangeBuilder()
                                .setReversed(true)
                                .setLimit(1)
                                .build()));
        for (Row<UUID, ByteBuffer> row : rows) {
            UUID dataId = row.getKey();
            for (Column<ByteBuffer> column : row.getColumns()) {
                resultMap.put(dataId, column.getName());
            }
        }
    }
    return resultMap;
}
 
Example #2
Source File: AstyanaxEventReaderDAO.java    From emodb with Apache License 2.0 6 votes vote down vote up
@Override
public Iterator<String> listChannels() {
    final Iterator<Row<String, ByteBuffer>> rowIter = execute(
            _keyspace.prepareQuery(ColumnFamilies.MANIFEST, ConsistencyLevel.CL_LOCAL_QUORUM)
                    .getAllRows()
                    .setRowLimit(1000)
                    .withColumnRange(new RangeBuilder().setLimit(1).build()))
            .iterator();
    return new AbstractIterator<String>() {
        @Override
        protected String computeNext() {
            while (rowIter.hasNext()) {
                Row<String, ByteBuffer> row = rowIter.next();
                if (!row.getColumns().isEmpty()) {
                    return row.getKey();
                }
            }
            return endOfData();
        }
    };
}
 
Example #3
Source File: AstyanaxManifestPersister.java    From emodb with Apache License 2.0 6 votes vote down vote up
@Override
public void delete(String channel, ByteBuffer slabId) {
    // Deletes don't need to be durable.  If a delete is lost, the next reader to come along and find no events
    // will execute the delete again.
    MutationBatch mutation = _keyspace.prepareMutationBatch(ConsistencyLevel.CL_ANY);

    mutation.withRow(ColumnFamilies.MANIFEST, channel)
            .deleteColumn(slabId);

    mutation.withRow(ColumnFamilies.SLAB, slabId)
            .delete();

    execute(mutation);

    _deleteMeter.mark();
}
 
Example #4
Source File: AstyanaxManifestPersister.java    From emodb with Apache License 2.0 6 votes vote down vote up
private void save(String channel, ByteBuffer slabId, boolean open, ConsistencyLevel consistency) {
    MutationBatch mutation = _keyspace.prepareMutationBatch(consistency);

    Duration ttl = getTtl(channel, open);
    mutation.withRow(ColumnFamilies.MANIFEST, channel)
            .putColumn(slabId, open, Ttls.toSeconds(ttl, 1, null));

    // Readers check for the open slab marker to see if a slab is open and may not re-read the manifest open
    // flag very often.  So delete the open slab marker so readers notice the state change more quickly.
    if (!open) {
        mutation.withRow(ColumnFamilies.SLAB, slabId)
                .deleteColumn(Constants.OPEN_SLAB_MARKER);
    }

    execute(mutation);
}
 
Example #5
Source File: AstyanaxQueueDAO.java    From emodb with Apache License 2.0 6 votes vote down vote up
@Override
public Iterator<ByteBuffer> scanRecords(UUID dataId, @Nullable ByteBuffer from, @Nullable final ByteBuffer to,
                                        int batchSize, int limit) {
    final Iterator<Column<ByteBuffer>> iter = executePaginated(
            _keyspace.prepareQuery(CF_DEDUP_DATA, ConsistencyLevel.CL_LOCAL_QUORUM)
                    .getKey(dataId)
                    .withColumnRange(new RangeBuilder()
                            .setStart(Objects.firstNonNull(from, EMPTY_BUFFER))
                            .setEnd(Objects.firstNonNull(to, EMPTY_BUFFER))
                            .setLimit(batchSize)
                            .build())
                    .autoPaginate(true));

    return Iterators.limit(new AbstractIterator<ByteBuffer>() {
        @Override
        protected ByteBuffer computeNext() {
            while (iter.hasNext()) {
                ByteBuffer record = iter.next().getName();
                if (!record.equals(to)) {  // To is exclusive
                    return record;
                }
            }
            return endOfData();
        }
    }, limit);
}
 
Example #6
Source File: ConsistencyTopologyAdapter.java    From emodb with Apache License 2.0 6 votes vote down vote up
/**
 * Reduce the desired consistency level to be compatible with the deployed ring topology.
 */
public ConsistencyLevel clamp(ConsistencyLevel consistencyLevel) {
    // Cassandra only allows the use of LOCAL_QUORUM and EACH_QUORUM if the keyspace
    // placement strategy is NetworkTopologyStrategy
    if ((consistencyLevel == ConsistencyLevel.CL_LOCAL_QUORUM || consistencyLevel == ConsistencyLevel.CL_EACH_QUORUM) && !_networkTopology) {
        consistencyLevel = ConsistencyLevel.CL_QUORUM;
    }
    if (consistencyLevel == ConsistencyLevel.CL_LOCAL_ONE && !_networkTopology) {
        consistencyLevel = ConsistencyLevel.CL_ONE;
    }

    // we may want to write to at two or three servers to ensure the write survives the
    // permanent failure of any single server.  but if the ring has fewer servers to
    // begin with (ie. it's a test ring) we must reduce the consistency level.
    if (consistencyLevel == ConsistencyLevel.CL_THREE && _replicationFactor < 3) {
        consistencyLevel = ConsistencyLevel.CL_TWO;
    }
    if (consistencyLevel == ConsistencyLevel.CL_TWO && _replicationFactor < 2) {
        consistencyLevel = ConsistencyLevel.CL_ONE;
    }

    return consistencyLevel;
}
 
Example #7
Source File: EntityCollectionManagerImpl.java    From usergrid with Apache License 2.0 6 votes vote down vote up
@Override
public Health getHealth() {

    try {
        ColumnFamily<String, String> CF_SYSTEM_LOCAL =
            new ColumnFamily<String, String>( "system.local", StringSerializer.get(), StringSerializer.get(),
                StringSerializer.get() );

        OperationResult<CqlResult<String, String>> result =
            keyspace.prepareQuery( CF_SYSTEM_LOCAL )
                .setConsistencyLevel(ConsistencyLevel.CL_ONE)
                .withCql( "SELECT now() FROM system.local;" )
                .execute();

        if ( result.getResult().getRows().size() > 0 ) {
            return Health.GREEN;
        }
    }
    catch ( ConnectionException ex ) {
        logger.error( "Error connecting to Cassandra", ex );
    }

    return Health.RED;
}
 
Example #8
Source File: AstyanaxQueueDAO.java    From emodb with Apache License 2.0 6 votes vote down vote up
@Override
public Iterator<String> listQueues() {
    final Iterator<Row<String, UUID>> rowIter = execute(
            _keyspace.prepareQuery(CF_DEDUP_MD, ConsistencyLevel.CL_LOCAL_QUORUM)
                    .getAllRows()
                    .setRowLimit(100)
                    .withColumnRange(new RangeBuilder().setLimit(1).build()))
            .iterator();
    return new AbstractIterator<String>() {
        @Override
        protected String computeNext() {
            while (rowIter.hasNext()) {
                Row<String, UUID> row = rowIter.next();
                if (!row.getColumns().isEmpty()) {
                    return row.getKey();
                }
            }
            return endOfData();
        }
    };
}
 
Example #9
Source File: MultiRowShardColumnIterator.java    From usergrid with Apache License 2.0 6 votes vote down vote up
public MultiRowShardColumnIterator( final Keyspace keyspace, final ColumnFamily<R, C> cf,
                                    final ConsistencyLevel consistencyLevel, final ColumnParser<C, T> columnParser,
                                    final ColumnSearch<T> columnSearch, final Comparator<T> comparator,
                                    final int pageSize, final List<SmartShard> rowKeysWithShardEnd,
                                    final boolean ascending, final Optional<Long> lastTimestamp) {
    this.cf = cf;
    this.pageSize = pageSize;
    this.columnParser = columnParser;
    this.columnSearch = columnSearch;
    this.comparator = comparator;
    this.keyspace = keyspace;
    this.consistencyLevel = consistencyLevel;
    this.moreToReturn = true;
    this.rowKeysWithShardEnd = rowKeysWithShardEnd;
    this.resultsTracking = new ArrayList<>();
    this.ascending = ascending;
    this.lastTimestamp = lastTimestamp;


}
 
Example #10
Source File: MultiRowColumnIterator.java    From usergrid with Apache License 2.0 6 votes vote down vote up
/**
 * Create the iterator
 */
public MultiRowColumnIterator( final Keyspace keyspace, final ColumnFamily<R, C> cf,
                               final ConsistencyLevel consistencyLevel, final ColumnParser<C, T> columnParser,
                               final ColumnSearch<T> columnSearch, final Comparator<T> comparator,
                               final Collection<R> rowKeys, final int pageSize ) {
    this.cf = cf;
    this.pageSize = pageSize;
    this.columnParser = columnParser;
    this.columnSearch = columnSearch;
    this.comparator = comparator;
    this.rowKeys = rowKeys;
    this.keyspace = keyspace;
    this.consistencyLevel = consistencyLevel;
    this.moreToReturn = true;

    //        seenResults = new HashMap<>( pageSize * 10 );
}
 
Example #11
Source File: AstyanaxStorageProvider.java    From emodb with Apache License 2.0 6 votes vote down vote up
private static void deleteDataColumns(AstyanaxTable table, String blobId, ColumnList<Composite> columns, ConsistencyLevel consistency, Long timestamp) {
    for (AstyanaxStorage storage : table.getWriteStorage()) {
        BlobPlacement placement = (BlobPlacement) storage.getPlacement();

        // Any columns with a timestamp older than the one we expect must be from an old version
        // of the blob.  This should be rare, but if it happens clean up and delete the old data.
        MutationBatch mutation = placement.getKeyspace().prepareMutationBatch(consistency);
        ColumnListMutation<Composite> row = mutation.withRow(
                placement.getBlobColumnFamily(), storage.getRowKey(blobId));
        boolean found = false;
        for (Column<Composite> column : columns) {
            if (null != timestamp && column.getTimestamp() < timestamp) {
                if (ColumnGroup.B.name().equals(column.getName().get(0, AsciiSerializer.get()))) {
                    int chunkId = column.getName().get(1, IntegerSerializer.get());
                    row.deleteColumn(getColumn(ColumnGroup.B, chunkId))
                            .deleteColumn(getColumn(ColumnGroup.Z, chunkId));
                    found = true;
                }
            }
        }
        if (found) {
            execute(mutation);
        }
    }
}
 
Example #12
Source File: CassandraMutagenImplTest.java    From mutagen-cassandra with Apache License 2.0 6 votes vote down vote up
private static void defineKeyspace() {
		context=new AstyanaxContext.Builder()
			.forKeyspace("mutagen_test")
			.withAstyanaxConfiguration(new AstyanaxConfigurationImpl()
//					.setDiscoveryType(NodeDiscoveryType.RING_DESCRIBE)
//					.setCqlVersion("3.0.0")
//					.setTargetCassandraVersion("1.2")
				.setDefaultReadConsistencyLevel(ConsistencyLevel.CL_QUORUM)
				.setDefaultWriteConsistencyLevel(ConsistencyLevel.CL_QUORUM)
			)
			.withConnectionPoolConfiguration(
				new ConnectionPoolConfigurationImpl("testPool")
				.setPort(9160)
				.setMaxConnsPerHost(1)
				.setSeeds("localhost")
			)
			.withConnectionPoolMonitor(new CountingConnectionPoolMonitor())
			.buildKeyspace(ThriftFamilyFactory.getInstance());

		context.start();
		keyspace=context.getClient();
	}
 
Example #13
Source File: SorConsistencies.java    From emodb with Apache License 2.0 5 votes vote down vote up
public static com.datastax.driver.core.ConsistencyLevel toCql(WriteConsistency consistency) {
    switch (consistency) {
        case NON_DURABLE:
            return com.datastax.driver.core.ConsistencyLevel.ANY;  // at least 1 node, allow hinted handoff
        case WEAK:
            return com.datastax.driver.core.ConsistencyLevel.TWO;  // usually at least 2 nodes, survives the permanent failure of any single node
        case STRONG:
            return com.datastax.driver.core.ConsistencyLevel.LOCAL_QUORUM;   // single data center quorum
        case GLOBAL:
            return com.datastax.driver.core.ConsistencyLevel.EACH_QUORUM;    // all data center quorum
        default:
            throw new UnsupportedOperationException(String.valueOf(consistency));
    }
}
 
Example #14
Source File: ShardsColumnIterator.java    From usergrid with Apache License 2.0 5 votes vote down vote up
public ShardsColumnIterator(final EdgeSearcher<R, C, T> searcher,
                            final MultiTenantColumnFamily<ScopedRowKey<R>, C> cf, final Keyspace keyspace,
                            final ConsistencyLevel consistencyLevel, final int pageSize,
                            final boolean smartShardSeekEnabled) {
    this.searcher = searcher;
    this.cf = cf;
    this.keyspace = keyspace;
    this.pageSize = pageSize;
    this.consistencyLevel = consistencyLevel;
    this.smartShardSeekEnabled = smartShardSeekEnabled;
}
 
Example #15
Source File: AstyanaxStorageProvider.java    From emodb with Apache License 2.0 5 votes vote down vote up
@Inject
public AstyanaxStorageProvider(@BlobReadConsistency ConsistencyLevel readConsistency, MetricRegistry metricRegistry) {
    _readConsistency = Objects.requireNonNull(readConsistency, "readConsistency");
    _tokenFactory = new ByteOrderedPartitioner().getTokenFactory();
    _blobReadMeter = metricRegistry.meter(getMetricName("blob-read"));
    _blobWriteMeter = metricRegistry.meter(getMetricName("blob-write"));
    _blobDeleteMeter = metricRegistry.meter(getMetricName("blob-delete"));
    _blobCopyMeter = metricRegistry.meter(getMetricName("blob-copy"));
    _blobMetadataReadMeter = metricRegistry.meter(getMetricName("blob-metadata-read"));
    _blobMetadataWriteMeter = metricRegistry.meter(getMetricName("blob-metadata-write"));
    _blobMetadataDeleteMeter = metricRegistry.meter(getMetricName("blob-metadata-delete"));
    _blobMetadataCopyMeter = metricRegistry.meter(getMetricName("blob-metadata-copy"));
    _scanBatchTimer = metricRegistry.timer(getMetricName("scan-batch"));
    _scanReadMeter = metricRegistry.meter(getMetricName("scan-reads"));
}
 
Example #16
Source File: SorConsistencies.java    From emodb with Apache License 2.0 5 votes vote down vote up
public static ConsistencyLevel toAstyanax(WriteConsistency consistency) {
    switch (consistency) {
        case NON_DURABLE:
            return ConsistencyLevel.CL_ANY;  // at least 1 node, allow hinted handoff
        case WEAK:
            return ConsistencyLevel.CL_TWO;  // usually at least 2 nodes, survives the permanent failure of any single node
        case STRONG:
            return ConsistencyLevel.CL_LOCAL_QUORUM;   // single data center quorum
        case GLOBAL:
            return ConsistencyLevel.CL_EACH_QUORUM;    // all data center quorum
        default:
            throw new UnsupportedOperationException(String.valueOf(consistency));
    }
}
 
Example #17
Source File: SorConsistencies.java    From emodb with Apache License 2.0 5 votes vote down vote up
public static com.datastax.driver.core.ConsistencyLevel toCql(ReadConsistency consistency) {
    switch (consistency) {
        case WEAK:
            return com.datastax.driver.core.ConsistencyLevel.LOCAL_ONE;
        case STRONG:
            return com.datastax.driver.core.ConsistencyLevel.LOCAL_QUORUM;
        default:
            throw new UnsupportedOperationException((String.valueOf(consistency)));
    }
}
 
Example #18
Source File: SorConsistencies.java    From emodb with Apache License 2.0 5 votes vote down vote up
public static ConsistencyLevel toAstyanax(ReadConsistency consistency) {
    switch (consistency) {
        case WEAK:
            return ConsistencyLevel.CL_LOCAL_ONE;           // first node to respond
        case STRONG:
            return ConsistencyLevel.CL_LOCAL_QUORUM;  // single data center quorum
        default:
            throw new UnsupportedOperationException(String.valueOf(consistency));
    }
}
 
Example #19
Source File: AstyanaxEventReaderDAO.java    From emodb with Apache License 2.0 5 votes vote down vote up
/**
 * Reads the ordered manifest for a channel.  The read can either be weak or strong.  A weak read will use CL1
 * and may use the cached oldest slab from a previous strong call to improve performance.  A strong read will use
 * CL local_quorum and will always read the entire manifest row.  This makes a weak read significantly faster than a
 * strong read but also means the call is not guaranteed to return the entire manifest.  Because of this at least
 * every 10 seconds a weak read for a channel is automatically promoted to a strong read.
 *
 * The vast majority of calls to this method are performed during a "peek" or "poll" operation.  Since these are
 * typically called repeatedly a weak call provides improved performance while guaranteeing that at least every
 * 10 seconds the manifest is strongly read so no slabs are missed over time.  Calls which must guarantee
 * the full manifest should explicitly request strong consistency.
 */
private Iterator<Column<ByteBuffer>> readManifestForChannel(final String channel, final boolean weak) {
    final ByteBuffer oldestSlab = weak ? _oldestSlab.getIfPresent(channel) : null;
    final ConsistencyLevel consistency;

    RangeBuilder range = new RangeBuilder().setLimit(50);
    if (oldestSlab != null) {
        range.setStart(oldestSlab);
        consistency = ConsistencyLevel.CL_LOCAL_ONE;
    } else {
        consistency = ConsistencyLevel.CL_LOCAL_QUORUM;
    }

    final Iterator<Column<ByteBuffer>> manifestColumns = executePaginated(
            _keyspace.prepareQuery(ColumnFamilies.MANIFEST, consistency)
                    .getKey(channel)
                    .withColumnRange(range.build())
                    .autoPaginate(true));

    if (oldestSlab != null) {
        // Query was executed weakly using the cached oldest slab, so don't update the cache with an unreliable oldest value
        return manifestColumns;
    } else {
        PeekingIterator<Column<ByteBuffer>> peekingManifestColumns = Iterators.peekingIterator(manifestColumns);
        if (peekingManifestColumns.hasNext()) {
            // Cache the first slab returned from querying the full manifest column family since it is the oldest.
            cacheOldestSlabForChannel(channel, TimeUUIDSerializer.get().fromByteBuffer(peekingManifestColumns.peek().getName()));
            return peekingManifestColumns;
        } else {
            // Channel was completely empty.  Cache a TimeUUID for the current time.  This will cause future calls
            // to read at most 1 minute of tombstones until the cache expires 10 seconds later.
            cacheOldestSlabForChannel(channel, TimeUUIDs.newUUID());
            return Iterators.emptyIterator();
        }
    }
}
 
Example #20
Source File: HystrixCassandraPut.java    From Nicobar with Apache License 2.0 5 votes vote down vote up
@Override
protected Void run() throws Exception {
    MutationBatch m = keyspace.prepareMutationBatch().setConsistencyLevel(ConsistencyLevel.CL_QUORUM);

    // Setting columns in a standard column
    ColumnListMutation<String> cm = m.withRow(columnFamily, rowKey);
    for (String key : attributes.keySet()) {
        Object o = attributes.get(key);
        if (o != null) {
            // unfortunately the 'putColumn' method does not nicely figure
            // out what type the Object is so we need to do it manually
            if (o instanceof String) {
                cm.putColumn(key, (String) o, ttlSeconds);
            } else if (o instanceof Boolean) {
                cm.putColumn(key, (Boolean) o, ttlSeconds);
            } else if (o instanceof Integer) {
                cm.putColumn(key, (Integer) o, ttlSeconds);
            } else if (o instanceof Long) {
                cm.putColumn(key, (Long) o, ttlSeconds);
            } else if (o instanceof Double) {
                cm.putColumn(key, (Double) o, ttlSeconds);
            } else if (o instanceof Date) {
                cm.putColumn(key, (Date) o, ttlSeconds);
            } else if (o instanceof byte[]) {
                cm.putColumn(key, (byte[]) o, ttlSeconds);
            } else if (o instanceof ByteBuffer) {
                cm.putColumn(key, (ByteBuffer) o, ttlSeconds);
            } else {
                throw new IllegalArgumentException("Unsupported object instance type: "
                        + o.getClass().getSimpleName());
            }
        }
    }
    m.execute();
    return null;
}
 
Example #21
Source File: BatchUpdate.java    From emodb with Apache License 2.0 5 votes vote down vote up
public BatchUpdate(CassandraKeyspace keyspace, ConsistencyLevel consistencyLevel, int maxRows, int maxColumns) {
    checkArgument(maxRows > 0);
    checkArgument(maxColumns > 0);
    _keyspace = checkNotNull(keyspace);
    _consistencyLevel = checkNotNull(consistencyLevel);
    _maxRows = maxRows;
    _maxColumns = maxColumns;
}
 
Example #22
Source File: AstyanaxStorageProvider.java    From emodb with Apache License 2.0 5 votes vote down vote up
@ParameterizedTimed(type = "AstyanaxStorageProvider")
    @Override
    public StorageSummary readMetadata(Table tbl, String blobId) {
        AstyanaxTable table = (AstyanaxTable) Objects.requireNonNull(tbl, "table");
        Objects.requireNonNull(blobId, "blobId");
        AstyanaxStorage storage = table.getReadStorage();
        BlobPlacement placement = (BlobPlacement) storage.getPlacement();

        // Do a column range query on all the A and B columns.  Don't get the Z columns with the binary data.
        Composite start = getColumnPrefix(ColumnGroup.A, Composite.ComponentEquality.LESS_THAN_EQUAL);
        Composite end = getColumnPrefix(ColumnGroup.B, Composite.ComponentEquality.GREATER_THAN_EQUAL);
        ColumnList<Composite> columns = execute(placement.getKeyspace()
                .prepareQuery(placement.getBlobColumnFamily(), _readConsistency)
                .getKey(storage.getRowKey(blobId))
                .withColumnRange(start, end, false, Integer.MAX_VALUE));

        StorageSummary summary = toStorageSummary(columns);
        if (summary == null) {
            return null;
        }

//      TODO should be removed for blob s3 migration
//      Cleanup older versions of the blob, if any (unlikely).
        deleteDataColumns(table, blobId, columns, ConsistencyLevel.CL_ANY, summary.getTimestamp());

        _blobMetadataReadMeter.mark();
        return summary;
    }
 
Example #23
Source File: AstyanaxStorageProvider.java    From emodb with Apache License 2.0 5 votes vote down vote up
private static Iterator<Map.Entry<String, StorageSummary>> decodeMetadataRows(
        final Iterator<Row<ByteBuffer, Composite>> rowIter, final AstyanaxTable table) {
    return new AbstractIterator<Map.Entry<String, StorageSummary>>() {
        @Override
        protected Map.Entry<String, StorageSummary> computeNext() {
            while (rowIter.hasNext()) {
                Row<ByteBuffer, Composite> row = rowIter.next();
                ByteBuffer key = row.getKey();
                ColumnList<Composite> columns = row.getColumns();

                String blobId = AstyanaxStorage.getContentKey(key);

                StorageSummary summary = toStorageSummary(columns);
                if (summary == null) {
                    continue;  // Partial blob, parts may still be replicating.
                }

                // TODO should be removed for blob s3 migration
                // Cleanup older versions of the blob, if any (unlikely).
                deleteDataColumns(table, blobId, columns, ConsistencyLevel.CL_ANY, summary.getTimestamp());

                return Maps.immutableEntry(blobId, summary);
            }
            return endOfData();
        }
    };
}
 
Example #24
Source File: AstyanaxQueueDAO.java    From emodb with Apache License 2.0 5 votes vote down vote up
@Override
public Map<UUID, String> loadSegments(String queue) {
    Map<UUID, String> resultMap = Maps.newHashMap();
    Iterator<Column<UUID>> iter = executePaginated(
            _keyspace.prepareQuery(CF_DEDUP_MD, ConsistencyLevel.CL_LOCAL_QUORUM)
                    .getKey(queue)
                    .withColumnRange(new RangeBuilder().setLimit(100).build())
                    .autoPaginate(true));
    while (iter.hasNext()) {
        Column<UUID> column = iter.next();
        resultMap.put(column.getName(), column.getStringValue());
    }
    return resultMap;
}
 
Example #25
Source File: AstyanaxQueueDAO.java    From emodb with Apache License 2.0 5 votes vote down vote up
@Nullable
@Override
public ByteBuffer findMinRecord(UUID dataId, @Nullable ByteBuffer from) {
    // Use a column range with a "start" to skip past tombstones.
    ColumnList<ByteBuffer> columns = execute(_keyspace.prepareQuery(CF_DEDUP_DATA, ConsistencyLevel.CL_LOCAL_QUORUM)
            .getKey(dataId)
            .withColumnRange(new RangeBuilder()
                    .setStart(Objects.firstNonNull(from, EMPTY_BUFFER))
                    .setLimit(1)
                    .build()));
    return !columns.isEmpty() ? columns.getColumnByIndex(0).getName() : null;
}
 
Example #26
Source File: AstyanaxEventWriterDAO.java    From emodb with Apache License 2.0 5 votes vote down vote up
@Override
public void delete(String channel, Collection<EventId> eventIds) {
    checkNotNull(channel, "channel");
    checkNotNull(eventIds, "eventIds");

    ListMultimap<ByteBuffer, Integer> eventsBySlab = ArrayListMultimap.create();
    for (EventId eventId : eventIds) {
        AstyanaxEventId eventIdImpl = (AstyanaxEventId) eventId;
        checkArgument(channel.equals(eventIdImpl.getChannel()));
        eventsBySlab.put(eventIdImpl.getSlabId(), eventIdImpl.getEventIdx());
    }

    // We might be able to use weak consistency since we're allowed to forget deletes and repeat events.  But we'd
    // need to measure it in production to see how frequently weak consistency would cause events to repeat.
    BatchUpdate update = new BatchUpdate(_keyspace, ConsistencyLevel.CL_LOCAL_QUORUM,
            Constants.MUTATION_MAX_ROWS, Constants.MUTATION_MAX_COLUMNS);

    for (Map.Entry<ByteBuffer, Collection<Integer>> entry : eventsBySlab.asMap().entrySet()) {
        ByteBuffer slabId = entry.getKey();
        Collection<Integer> eventIdxs = entry.getValue();

        BatchUpdate.Row<ByteBuffer, Integer> row = update.updateRow(ColumnFamilies.SLAB, slabId);
        for (Integer eventIdx : eventIdxs) {
            row.deleteColumn(eventIdx);
        }
    }

    update.finish();
}
 
Example #27
Source File: AstyanaxManifestPersister.java    From emodb with Apache License 2.0 5 votes vote down vote up
@Override
public void open(String channel, ByteBuffer slabId) {
    // Updates on open must be durable or we'll lose slabs.
    save(channel, slabId, true, ConsistencyLevel.CL_LOCAL_QUORUM);

    _openMeter.mark();
}
 
Example #28
Source File: AstyanaxEventReaderDAO.java    From emodb with Apache License 2.0 5 votes vote down vote up
@Override
public boolean moveIfFast(String fromChannel, String toChannel) {
    Iterator<Column<ByteBuffer>> manifestColumns = executePaginated(
            _keyspace.prepareQuery(ColumnFamilies.MANIFEST, ConsistencyLevel.CL_LOCAL_QUORUM)
                    .getKey(fromChannel)
                    .withColumnRange(new RangeBuilder().setLimit(50).build())
                    .autoPaginate(true));

    List<ByteBuffer> closedSlabs = Lists.newArrayList();
    boolean movedAll = true;
    while (manifestColumns.hasNext()) {
        Column<ByteBuffer> manifestColumn = manifestColumns.next();
        ByteBuffer slabId = manifestColumn.getName();
        boolean open = manifestColumn.getBooleanValue();
        if (open) {
            // Can't safely re-assign open slabs to another channel since writers may still be writing.
            movedAll = false;  // All events in the open slab might be deleted, but don't check for that here.
            continue;
        }
        closedSlabs.add(slabId);
        if (closedSlabs.size() >= SLAB_MOVE_BATCH) {
            _manifestPersister.move(fromChannel, toChannel, closedSlabs, false);
            closedSlabs.clear();
        }
    }
    if (!closedSlabs.isEmpty()) {
        _manifestPersister.move(fromChannel, toChannel, closedSlabs, false);
    }

    return movedAll;
}
 
Example #29
Source File: AstyanaxManifestPersister.java    From emodb with Apache License 2.0 5 votes vote down vote up
@Override
public void close(String channel, ByteBuffer slabId) {
    // Updates on close don't need to be particularly durable since open slabs will time out automatically if
    // not closed explicitly.
    save(channel, slabId, false, ConsistencyLevel.CL_ANY);

    // Don't meter close calls.  Callers can track close calls w/metrics specific to the reason for the close.
}
 
Example #30
Source File: BlobStoreModule.java    From emodb with Apache License 2.0 4 votes vote down vote up
@Provides @Singleton @BlobReadConsistency
ConsistencyLevel provideBlobReadConsistency(BlobStoreConfiguration configuration) {
    // By default use local quorum
    return Optional.fromNullable(configuration.getReadConsistency()).or(ConsistencyLevel.CL_LOCAL_QUORUM);
}