com.amazonaws.services.dynamodbv2.model.ScanResult#getLastEvaluatedKey

Source File: ScanRecordReadRequest.java From emr-dynamodb-connector with Apache License 2.0

6 votes

@Override
protected PageResults<Map<String, AttributeValue>> fetchPage(RequestLimit lim) {
  // Read from DynamoDB
  RetryResult<ScanResult> retryResult = context.getClient().scanTable(tableName, null, segment,
      context.getSplit().getTotalSegments(), lastEvaluatedKey, lim.items, context.getReporter());

  ScanResult result = retryResult.result;
  int retries = retryResult.retries;

  double consumedCapacityUnits = 0.0;
  if (result.getConsumedCapacity() != null) {
    consumedCapacityUnits = result.getConsumedCapacity().getCapacityUnits();
  }
  return new PageResults<>(result.getItems(), result.getLastEvaluatedKey(), consumedCapacityUnits,
      retries);
}

Source File: ScanSegmentWorker.java From dynamodb-janusgraph-storage-backend with Apache License 2.0

6 votes

@SuppressFBWarnings(value = "IT_NO_SUCH_ELEMENT",
    justification = "https://github.com/awslabs/dynamodb-janusgraph-storage-backend/issues/222")
@Override
public ScanResult next() {
    final Scan backoff = new Scan(request, delegate, lastConsumedCapacity);
    ScanResult result = null;
    try {
        result = backoff.runWithBackoff(); //this will be non-null or runWithBackoff throws
    } catch (BackendException e) {
        throw new BackendRuntimeException(e);
    }

    if (result.getConsumedCapacity() != null) {
        lastConsumedCapacity = result.getConsumedCapacity().getCapacityUnits().intValue();
    }

    if (result.getLastEvaluatedKey() != null && !result.getLastEvaluatedKey().isEmpty()) {
        hasNext = true;
        request.setExclusiveStartKey(result.getLastEvaluatedKey());
    } else {
        hasNext = false;
    }

    return result;
}

Source File: TransformerHolisticTests.java From aws-dynamodb-encryption-java with Apache License 2.0

6 votes

private void dumpTables() {
    for (String table : client.listTables().getTableNames()) {
        ScanResult scanResult;
        Map<String, AttributeValue> lastKey = null;
        do {
            scanResult = client.scan(new ScanRequest().withTableName(table).withExclusiveStartKey(lastKey));
            lastKey = scanResult.getLastEvaluatedKey();
            for (Map<String, AttributeValue> map : scanResult.getItems()) {
                for (Map.Entry<String, AttributeValue> item : map.entrySet()) {
                    System.out.print("item.put(\"");
                    System.out.print(item.getKey());
                    System.out.print("\", b642Av(\"");
                    System.out.print(Base64.encodeAsString(AttributeValueMarshaller.marshall(item.getValue()).array()));
                    System.out.println("\"));");
                }
                System.out.print("ddb.putItem(new PutItemRequest(\"");
                System.out.print(table);
                System.out.println("\", item));");
                System.out.println("item.clear();");
                System.out.println();
            }
        } while (lastKey != null);

    }
}

Source File: DynamoDBOperations.java From geowave with Apache License 2.0

6 votes

public Iterator<GeoWaveRow> getRowsFromDataIndex(final short adapterId, final String typeName) {
  final List<GeoWaveRow> resultList = new ArrayList<>();
  // fill result list
  ScanResult result =
      getResults(
          typeName + "_" + getQualifiedTableName(DataIndexUtils.DATA_ID_INDEX.getName()),
          adapterId,
          resultList,
          null);
  while ((result.getLastEvaluatedKey() != null) && !result.getLastEvaluatedKey().isEmpty()) {
    result =
        getResults(
            typeName + "_" + getQualifiedTableName(DataIndexUtils.DATA_ID_INDEX.getName()),
            adapterId,
            resultList,
            result.getLastEvaluatedKey());
  }
  return resultList.iterator();
}

Source File: GenericDynamoDB.java From strongbox with Apache License 2.0

5 votes

private Stream<Entry> scan(SecretEventStream.Filter<Entry> filter, Converters converters) {
    ScanRequest scanRequest = new ScanRequest();
    scanRequest.withConsistentRead(true);
    scanRequest.withTableName(tableName);

    FilterGenerator filterGenerator = new FilterGenerator();
    FilterGenerator.Filter generated = filterGenerator.process(filter.parsedAttributeCondition.get(), converters);

    if(!generated.expressionAttributeNames.isEmpty()) {
        scanRequest.withExpressionAttributeNames(generated.expressionAttributeNames);
    }

    if (!generated.expressionAttributeValues.isEmpty()) {
        scanRequest.withExpressionAttributeValues(generated.expressionAttributeValues);
    }

    scanRequest.withFilterExpression(generated.filterExpression);

    ScanResult result = client.scan(scanRequest);

    List<Map<String, AttributeValue>> results = new ArrayList<>();
    results.addAll(result.getItems());

    while (result.getLastEvaluatedKey() != null) {
        scanRequest = scanRequest.withExclusiveStartKey(result.getLastEvaluatedKey());

        result = client.scan(scanRequest);

        results.addAll(result.getItems());
    }

    Stream<Entry> typedResult = results.stream().map(this::fromMap);

    if (filter.reverse) {
        typedResult = Lists.reverse(typedResult.collect(Collectors.toCollection(LinkedList::new))).stream();
    }

    return typedResult;
}

Source File: LowLevelParallelScan.java From aws-doc-sdk-examples with Apache License 2.0

5 votes

@Override
public void run() {
    System.out.println("Scanning " + tableName + " segment " + segment + " out of " + totalSegments
        + " segments " + itemLimit + " items at a time...");
    Map<String, AttributeValue> exclusiveStartKey = null;
    int totalScannedItemCount = 0;
    int totalScanRequestCount = 0;
    try {
        while (true) {
            ScanRequest scanRequest = new ScanRequest().withTableName(tableName).withLimit(itemLimit)
                .withExclusiveStartKey(exclusiveStartKey).withTotalSegments(totalSegments).withSegment(segment);

            ScanResult result = client.scan(scanRequest);

            totalScanRequestCount++;
            totalScannedItemCount += result.getScannedCount();

            // print items returned from scan request
            processScanResult(segment, result);

            exclusiveStartKey = result.getLastEvaluatedKey();
            if (exclusiveStartKey == null) {
                break;
            }
        }
    }
    catch (AmazonServiceException ase) {
        System.err.println(ase.getMessage());
    }
    finally {
        System.out.println("Scanned " + totalScannedItemCount + " items from segment " + segment + " out of "
            + totalSegments + " of " + tableName + " with " + totalScanRequestCount + " scan requests");
    }
}

Source File: ScanSegmentWorker.java From dynamodb-import-export-tool with Apache License 2.0

5 votes

@Override
public SegmentedScanResult call() {
    ScanResult result = null;
    result = runWithBackoff();

    final ConsumedCapacity cc = result.getConsumedCapacity();

    if (cc != null && cc.getCapacityUnits() != null) {
        lastConsumedCapacity = result.getConsumedCapacity()
                .getCapacityUnits().intValue();
    } else if (result.getScannedCount() != null && result.getCount() != null) {

        final boolean isConsistent = request.getConsistentRead();
        int itemSize = isConsistent ? BootstrapConstants.STRONGLY_CONSISTENT_READ_ITEM_SIZE
                : BootstrapConstants.EVENTUALLY_CONSISTENT_READ_ITEM_SIZE;

        lastConsumedCapacity = (result.getScannedCount() / (int) Math.max(1.0, result.getCount()))
                * (ItemSizeCalculator.calculateScanResultSizeInBytes(result) / itemSize);
    }

    if (result.getLastEvaluatedKey() != null
            && !result.getLastEvaluatedKey().isEmpty()) {
        hasNext = true;
        request.setExclusiveStartKey(result.getLastEvaluatedKey());
    } else {
        hasNext = false;
    }

    if (lastConsumedCapacity > 0) {
        rateLimiter.acquire(lastConsumedCapacity);
    }
    return new SegmentedScanResult(result, request.getSegment());
}

Source File: VerifyIndex.java From aws-big-data-blog with Apache License 2.0

5 votes

private void fetchIndexList() {
   final ScanRequest req = new ScanRequest("s3-index-example.mikedeck-index").withAttributesToGet("Key");
   ScanResult result = DDB.scan(req);
   addIndexItems(result.getItems());
   while (result.getLastEvaluatedKey() != null && !result.getLastEvaluatedKey().isEmpty()) {
      req.setExclusiveStartKey(result.getLastEvaluatedKey());
      result = DDB.scan(req);
      addIndexItems(result.getItems());
   }
}

Source File: DynamoDBService.java From Doradus with Apache License 2.0

5 votes

@Override
public List<String> getRows(String storeName, String continuationToken, int count) {
    String tableName = storeToTableName(storeName);
    ScanRequest scanRequest = new ScanRequest(tableName);
    scanRequest.setAttributesToGet(Arrays.asList(ROW_KEY_ATTR_NAME)); // attributes to get
    if (continuationToken != null) {
        scanRequest.setExclusiveStartKey(makeDDBKey(continuationToken));
    }
    List<String> rowKeys = new ArrayList<>();
    while (rowKeys.size() < count) {
        ScanResult scanResult = scan(scanRequest);
        List<Map<String, AttributeValue>> itemList = scanResult.getItems();
        if (itemList.size() == 0) {
            break;
        }
        for (Map<String, AttributeValue> attributeMap : itemList) {
            AttributeValue rowAttr = attributeMap.get(ROW_KEY_ATTR_NAME);
            rowKeys.add(rowAttr.getS());
            if (rowKeys.size() >= count) {
                break;
            }
        }
        Map<String, AttributeValue> lastEvaluatedKey = scanResult.getLastEvaluatedKey();
        if (lastEvaluatedKey == null) {
            break;
        }
        scanRequest.setExclusiveStartKey(lastEvaluatedKey);
    }
    return rowKeys;
}

Source File: DynamoDBService2.java From Doradus with Apache License 2.0

5 votes

@Override
public List<String> getRows(String storeName, String continuationToken, int count) {
	Timer t = new Timer();
	ScanRequest scanRequest = new ScanRequest(getTenant().getName());
    scanRequest.setAttributesToGet(Arrays.asList("key")); // attributes to get
    //if (continuationToken != null) {
    //	scanRequest.setExclusiveStartKey(getPrimaryKey(storeName + "_" + continuationToken, "\u007F"));
    //} else {
    //	scanRequest.setExclusiveStartKey(getPrimaryKey(storeName + "_", "\0"));
    //}

    Set<String> rowKeys = new HashSet<>();
    while (rowKeys.size() < count) {
        ScanResult scanResult = m_client.scan(scanRequest);
        List<Map<String, AttributeValue>> itemList = scanResult.getItems();
        if (itemList.size() == 0) break;
        for (Map<String, AttributeValue> attributeMap : itemList) {
            AttributeValue rowAttr = attributeMap.get("key");
            if(!rowAttr.getS().startsWith(storeName)) continue;
            String name = rowAttr.getS().substring(storeName.length() + 1);
            if(continuationToken != null && continuationToken.compareTo(name) >= 0) continue;
            rowKeys.add(name);
        }
        Map<String, AttributeValue> lastEvaluatedKey = scanResult.getLastEvaluatedKey();
        if (lastEvaluatedKey == null) break;
    	scanRequest.setExclusiveStartKey(getPrimaryKey(lastEvaluatedKey.get("key").getS(), "\u007F"));
    }
    List<String> list = new ArrayList<>(rowKeys);
    Collections.sort(list);
	m_logger.debug("get rows in {} in {}", storeName, t);
    return list;
}

Source File: LowLevelParallelScan.java From aws-dynamodb-examples with Apache License 2.0

5 votes

@Override
public void run() {
    System.out.println("Scanning " + tableName + " segment " + segment + " out of " + totalSegments + " segments " + itemLimit + " items at a time...");
    Map<String, AttributeValue> exclusiveStartKey = null;
    int totalScannedItemCount = 0;
    int totalScanRequestCount = 0;
    try {
        while(true) {
            ScanRequest scanRequest = new ScanRequest()
                .withTableName(tableName)
                .withLimit(itemLimit)
                .withExclusiveStartKey(exclusiveStartKey)
                .withTotalSegments(totalSegments)
                .withSegment(segment);
            
            ScanResult result = client.scan(scanRequest);
            
            totalScanRequestCount++;
            totalScannedItemCount += result.getScannedCount();
            
            // print items returned from scan request
            processScanResult(segment, result);
            
            exclusiveStartKey = result.getLastEvaluatedKey();
            if (exclusiveStartKey == null) {
                break;
            }
        }
    } catch (AmazonServiceException ase) {
        System.err.println(ase.getMessage());
    } finally {
        System.out.println("Scanned " + totalScannedItemCount + " items from segment " + segment + " out of " + totalSegments + " of " + tableName + " with " + totalScanRequestCount + " scan requests");
    }
}

Source File: GenericDynamoDB.java From strongbox with Apache License 2.0

4 votes

private Stream<Entry> all() {
    ScanRequest scanRequest = new ScanRequest();
    scanRequest.withConsistentRead(true);
    scanRequest.withTableName(tableName);

    ScanResult result = client.scan(scanRequest);

    List<Map<String, AttributeValue>> results = new ArrayList<>();
    results.addAll(result.getItems());

    while (result.getLastEvaluatedKey() != null) {
        scanRequest = scanRequest.withExclusiveStartKey(result.getLastEvaluatedKey());

        result = client.scan(scanRequest);

        results.addAll(result.getItems());
    }


    return results.stream().map(this::fromMap);
}

Java Code Examples for com.amazonaws.services.dynamodbv2.model.ScanResult#getLastEvaluatedKey()