com.amazonaws.services.dynamodbv2.model.ScanResult Java Examples

@Override
protected PageResults<Map<String, AttributeValue>> fetchPage(RequestLimit lim) {
  // Read from DynamoDB
  RetryResult<ScanResult> retryResult = context.getClient().scanTable(tableName, null, segment,
      context.getSplit().getTotalSegments(), lastEvaluatedKey, lim.items, context.getReporter());

  ScanResult result = retryResult.result;
  int retries = retryResult.retries;

  double consumedCapacityUnits = 0.0;
  if (result.getConsumedCapacity() != null) {
    consumedCapacityUnits = result.getConsumedCapacity().getCapacityUnits();
  }
  return new PageResults<>(result.getItems(), result.getLastEvaluatedKey(), consumedCapacityUnits,
      retries);
}
 

/**
 * @return the list of device ID (UID) stored in the identity table.
 */
public List<String> listDevices() {
    List<String> devices = new ArrayList<String>(1000);

    ScanResult result = ddb.scan(new ScanRequest().withTableName(DEVICE_TABLE).withLimit(1000));

    for (Map<String, AttributeValue> item : result.getItems()) {
        String s = "";

        for (Entry<String, AttributeValue> entry : item.entrySet()) {
            s += " ** " + entry.getKey() + " = " + entry.getValue().getS();
        }

        devices.add(s);
    }

    return devices;
}
 

private static void findProductsForPriceLessThanZero() {
    
    Map<String, AttributeValue> expressionAttributeValues = new HashMap<String, AttributeValue>();
    expressionAttributeValues.put(":pr", new AttributeValue().withN("100"));
    
    ScanRequest scanRequest = new ScanRequest()
        .withTableName(tableName)
        .withFilterExpression("Price < :pr")
        .withExpressionAttributeValues(expressionAttributeValues)
        .withProjectionExpression("Id, Title, ProductCategory, Price");

    ScanResult result = client.scan(scanRequest);
    
    System.out.println("Scan of " + tableName + " for items with a price less than 100.");
    for (Map<String, AttributeValue> item : result.getItems()) {
        System.out.println("");
        printItem(item);
    }
}
 

/**
 * Returns the list of usernames stored in the identity table.
 * 
 * @return list of existing usernames in DynamoDB table
 */
public List<String> listUsers() {
    List<String> users = new ArrayList<String>(1000);

    ScanResult result = ddb.scan(new ScanRequest().withTableName(USER_TABLE).withLimit(1000));

    for (Map<String, AttributeValue> item : result.getItems()) {
        String s = "";

        for (Entry<String, AttributeValue> entry : item.entrySet()) {
            s += " ** " + entry.getKey() + " = " + entry.getValue().getS();
        }

        users.add(s);
    }

    return users;
}
 

/**
 * @return the list of device ID (UID) stored in the identity table.
 */
public List<String> listDevices() {
    List<String> devices = new ArrayList<String>(1000);

    ScanResult result = ddb.scan(new ScanRequest().withTableName(DEVICE_TABLE).withLimit(1000));

    for (Map<String, AttributeValue> item : result.getItems()) {
        String s = "";

        for (Entry<String, AttributeValue> entry : item.entrySet()) {
            s += " ** " + entry.getKey() + " = " + entry.getValue().getS();
        }

        devices.add(s);
    }

    return devices;
}
 

private ScanResult getResults(
    final String tableName,
    final short adapterId,
    final List<GeoWaveRow> resultList,
    final Map<String, AttributeValue> lastEvaluatedKey) {
  final ScanRequest request = new ScanRequest(tableName);
  if ((lastEvaluatedKey != null) && !lastEvaluatedKey.isEmpty()) {
    request.setExclusiveStartKey(lastEvaluatedKey);
  }
  final ScanResult result = client.scan(request);
  result.getItems().forEach(objMap -> {
    final byte[] dataId = objMap.get(DynamoDBRow.GW_PARTITION_ID_KEY).getB().array();
    final AttributeValue valueAttr = objMap.get(DynamoDBRow.GW_VALUE_KEY);
    final byte[] value = valueAttr == null ? null : valueAttr.getB().array();
    final AttributeValue visAttr = objMap.get(DynamoDBRow.GW_VISIBILITY_KEY);
    final byte[] vis = visAttr == null ? new byte[0] : visAttr.getB().array();

    resultList.add(DataIndexUtils.deserializeDataIndexRow(dataId, adapterId, value, vis));
  });
  return result;
}
 

public Iterator<GeoWaveRow> getRowsFromDataIndex(final short adapterId, final String typeName) {
  final List<GeoWaveRow> resultList = new ArrayList<>();
  // fill result list
  ScanResult result =
      getResults(
          typeName + "_" + getQualifiedTableName(DataIndexUtils.DATA_ID_INDEX.getName()),
          adapterId,
          resultList,
          null);
  while ((result.getLastEvaluatedKey() != null) && !result.getLastEvaluatedKey().isEmpty()) {
    result =
        getResults(
            typeName + "_" + getQualifiedTableName(DataIndexUtils.DATA_ID_INDEX.getName()),
            adapterId,
            resultList,
            result.getLastEvaluatedKey());
  }
  return resultList.iterator();
}
 

public RetryResult<ScanResult> scanTable(
    String tableName, DynamoDBQueryFilter dynamoDBQueryFilter, Integer segment, Integer
    totalSegments, Map<String, AttributeValue> exclusiveStartKey, long limit, Reporter reporter) {
  final ScanRequest scanRequest = new ScanRequest(tableName)
      .withExclusiveStartKey(exclusiveStartKey)
      .withLimit(Ints.checkedCast(limit))
      .withSegment(segment)
      .withTotalSegments(totalSegments)
      .withReturnConsumedCapacity(ReturnConsumedCapacity.TOTAL);

  if (dynamoDBQueryFilter != null) {
    Map<String, Condition> scanFilter = dynamoDBQueryFilter.getScanFilter();
    if (!scanFilter.isEmpty()) {
      scanRequest.setScanFilter(scanFilter);
    }
  }

  RetryResult<ScanResult> retryResult = getRetryDriver().runWithRetry(new Callable<ScanResult>() {
    @Override
    public ScanResult call() {
      log.debug("Executing DynamoDB scan: " + scanRequest);
      return dynamoDB.scan(scanRequest);
    }
  }, reporter, PrintCounter.DynamoDBReadThrottle);
  return retryResult;
}
 

private void dumpTables() {
    for (String table : client.listTables().getTableNames()) {
        ScanResult scanResult;
        Map<String, AttributeValue> lastKey = null;
        do {
            scanResult = client.scan(new ScanRequest().withTableName(table).withExclusiveStartKey(lastKey));
            lastKey = scanResult.getLastEvaluatedKey();
            for (Map<String, AttributeValue> map : scanResult.getItems()) {
                for (Map.Entry<String, AttributeValue> item : map.entrySet()) {
                    System.out.print("item.put(\"");
                    System.out.print(item.getKey());
                    System.out.print("\", b642Av(\"");
                    System.out.print(Base64.encodeAsString(AttributeValueMarshaller.marshall(item.getValue()).array()));
                    System.out.println("\"));");
                }
                System.out.print("ddb.putItem(new PutItemRequest(\"");
                System.out.print(table);
                System.out.println("\", item));");
                System.out.println("item.clear();");
                System.out.println();
            }
        } while (lastKey != null);

    }
}
 

private static void findProductsForPriceLessThanZero() {

        Map<String, AttributeValue> expressionAttributeValues = new HashMap<String, AttributeValue>();
        expressionAttributeValues.put(":pr", new AttributeValue().withN("100"));

        ScanRequest scanRequest = new ScanRequest().withTableName(tableName).withFilterExpression("Price < :pr")
            .withExpressionAttributeValues(expressionAttributeValues)
            .withProjectionExpression("Id, Title, ProductCategory, Price");

        ScanResult result = client.scan(scanRequest);

        System.out.println("Scan of " + tableName + " for items with a price less than 100.");
        for (Map<String, AttributeValue> item : result.getItems()) {
            System.out.println("");
            printItem(item);
        }
    }
 

/**
 * begins a scan with an exponential back off if throttled.
 */
public ScanResult runWithBackoff() {
    ScanResult result = null;
    boolean interrupted = false;
    try {
        do {
            try {
                result = client.scan(request);
            } catch (Exception e) {
                try {
                    Thread.sleep(exponentialBackoffTime);
                } catch (InterruptedException ie) {
                    interrupted = true;
                } finally {
                    exponentialBackoffTime *= 2;
                }
                continue;
            }
        } while (result == null);
        return result;
    } finally {
        if (interrupted) {
            Thread.currentThread().interrupt();
        }
    }
}
 

@SuppressFBWarnings(value = "IT_NO_SUCH_ELEMENT",
    justification = "https://github.com/awslabs/dynamodb-janusgraph-storage-backend/issues/222")
@Override
public ScanResult next() {
    final Scan backoff = new Scan(request, delegate, lastConsumedCapacity);
    ScanResult result = null;
    try {
        result = backoff.runWithBackoff(); //this will be non-null or runWithBackoff throws
    } catch (BackendException e) {
        throw new BackendRuntimeException(e);
    }

    if (result.getConsumedCapacity() != null) {
        lastConsumedCapacity = result.getConsumedCapacity().getCapacityUnits().intValue();
    }

    if (result.getLastEvaluatedKey() != null && !result.getLastEvaluatedKey().isEmpty()) {
        hasNext = true;
        request.setExclusiveStartKey(result.getLastEvaluatedKey());
    } else {
        hasNext = false;
    }

    return result;
}
 

public ScanResult scan(final ScanRequest request, final int permitsToConsume) throws BackendException {
    setUserAgent(request);
    ScanResult result;
    timedReadThrottle(SCAN, request.getTableName(), permitsToConsume);

    final Timer.Context apiTimerContext = getTimerContext(SCAN, request.getTableName());
    try {
        result = client.scan(request);
    } catch (Exception e) {
        throw processDynamoDbApiException(e, SCAN, request.getTableName());
    } finally {
        apiTimerContext.stop();
    }
    meterConsumedCapacity(SCAN, result.getConsumedCapacity());
    measureItemCount(SCAN, request.getTableName(), result.getCount());
    return result;
}
 

/**
 * @return the list of device ID (UID) stored in the identity table.
 */
public List<String> listDevices() {
    List<String> devices = new ArrayList<String>(1000);

    ScanResult result = ddb.scan(new ScanRequest().withTableName(DEVICE_TABLE).withLimit(1000));

    for (Map<String, AttributeValue> item : result.getItems()) {
        String s = "";

        for (Entry<String, AttributeValue> entry : item.entrySet()) {
            s += " ** " + entry.getKey() + " = " + entry.getValue().getS();
        }

        devices.add(s);
    }

    return devices;
}
 

/**
 * Returns the list of usernames stored in the identity table.
 * 
 * @return list of existing usernames in DynamoDB table
 */
public List<String> listUsers() {
    List<String> users = new ArrayList<String>(1000);

    ScanResult result = ddb.scan(new ScanRequest().withTableName(USER_TABLE).withLimit(1000));

    for (Map<String, AttributeValue> item : result.getItems()) {
        String s = "";

        for (Entry<String, AttributeValue> entry : item.entrySet()) {
            s += " ** " + entry.getKey() + " = " + entry.getValue().getS();
        }

        users.add(s);
    }

    return users;
}
 

/**
 * Get the next query data If the last request is equal to null then we have no more query
 * requests to fire
 *
 * <p> If asyncQueryResults is not empty, we have already fetched the next query data that can be
 * read immediately
 *
 * <p> If due to max async query limit, we couldn't fire async requests, we fire the request now
 */
@Override
protected Iterator<? extends Map<String, AttributeValue>> nextIterator(final int arg0) {

  synchronized (monitorLock) {
    if ((lastRequest == null) && asyncScanResults.isEmpty()) {
      return null;
    }

    ScanResult result = null;
    if ((lastRequest != null) && (asyncRequestsInProgress == 0)) {
      makeAsyncScan();
    }

    while (asyncScanResults.isEmpty()) {
      try {
        monitorLock.wait();
      } catch (final InterruptedException e) {
        LOGGER.error("Exception in Async paginated query " + e);
        e.printStackTrace();
      }
    }
    result = asyncScanResults.remove();

    return result == null ? null : result.getItems().iterator();
  }
}
 

private ScanResult scan(ScanRequest scanRequest) {
    m_logger.debug("Performing scan() request on table {}", scanRequest.getTableName());
    
    Timer timer = new Timer();
    boolean bSuccess = false;
    ScanResult scanResult = null;
    for (int attempts = 1; !bSuccess; attempts++) {
        try {
            scanResult = m_ddbClient.scan(scanRequest);
            if (attempts > 1) {
                m_logger.info("scan() succeeded on attempt #{}", attempts);
            }
            bSuccess = true;
            m_logger.debug("Time to scan table {}: {}", scanRequest.getTableName(), timer.toString());
        } catch (ProvisionedThroughputExceededException e) {
            if (attempts >= m_max_read_attempts) {
                String errMsg = "All retries exceeded; abandoning scan() for table: " + scanRequest.getTableName();
                m_logger.error(errMsg, e);
                throw new RuntimeException(errMsg, e);
            }
            m_logger.warn("scan() attempt #{} failed: {}", attempts, e);
            try {
                Thread.sleep(attempts * m_retry_wait_millis);
            } catch (InterruptedException ex2) {
                // ignore
            }
        }
    }
    return scanResult;
}
 

/**
 * Returns the list of usernames stored in the user table.
 */
public List<String> listUsers(String userTable) {
    List<String> users = new ArrayList<String>(1000);

    ScanResult result = ddb.scan(new ScanRequest().withTableName(userTable).withLimit(1000));

    for (Map<String, AttributeValue> item : result.getItems()) {
        users.add(item.get("username").getS());
    }

    return users;
}
 

@Override
public final Response scan(final Request request) {

    final String projectionExpression = request.getPartitionKey() + ", " + request.getSortKey();
    ScanRequest scanRequest = new ScanRequest()
            .withTableName(request.getTableName())
            .withProjectionExpression(projectionExpression);

    StringBuilder filterExpressionBuilder;
    Map<String, AttributeValue> expressionAttributeValues;
    if (request.getFilterData() != null) {
        filterExpressionBuilder = new StringBuilder();
        expressionAttributeValues = new HashMap<>();
        processFilterData(request, filterExpressionBuilder, expressionAttributeValues);
        // Add to ScanRequest.
        scanRequest.withFilterExpression(filterExpressionBuilder.toString());
        scanRequest.withExpressionAttributeValues(expressionAttributeValues);
    }



    final ScanResult scanResult = dynamoDBClient.scan(scanRequest);

    final StringBuilder response = new StringBuilder();
    response.append("PK of items read with scan (V2): ");
    for (Map<String, AttributeValue> item : scanResult.getItems()) {
        response.append(prepareKeyStr(item, request));
    }

    return new Response(response.toString(), null);
}
 

@Override
public List<String> getRows(String storeName, String continuationToken, int count) {
	Timer t = new Timer();
	ScanRequest scanRequest = new ScanRequest(getTenant().getName());
    scanRequest.setAttributesToGet(Arrays.asList("key")); // attributes to get
    //if (continuationToken != null) {
    //	scanRequest.setExclusiveStartKey(getPrimaryKey(storeName + "_" + continuationToken, "\u007F"));
    //} else {
    //	scanRequest.setExclusiveStartKey(getPrimaryKey(storeName + "_", "\0"));
    //}

    Set<String> rowKeys = new HashSet<>();
    while (rowKeys.size() < count) {
        ScanResult scanResult = m_client.scan(scanRequest);
        List<Map<String, AttributeValue>> itemList = scanResult.getItems();
        if (itemList.size() == 0) break;
        for (Map<String, AttributeValue> attributeMap : itemList) {
            AttributeValue rowAttr = attributeMap.get("key");
            if(!rowAttr.getS().startsWith(storeName)) continue;
            String name = rowAttr.getS().substring(storeName.length() + 1);
            if(continuationToken != null && continuationToken.compareTo(name) >= 0) continue;
            rowKeys.add(name);
        }
        Map<String, AttributeValue> lastEvaluatedKey = scanResult.getLastEvaluatedKey();
        if (lastEvaluatedKey == null) break;
    	scanRequest.setExclusiveStartKey(getPrimaryKey(lastEvaluatedKey.get("key").getS(), "\u007F"));
    }
    List<String> list = new ArrayList<>(rowKeys);
    Collections.sort(list);
	m_logger.debug("get rows in {} in {}", storeName, t);
    return list;
}
 

@Override
public Void call() {
    final ScanResult scanResult = result.getScanResult();
    final List<Map<String, AttributeValue>> items = scanResult.getItems();
    final Iterator<Map<String, AttributeValue>> it = items.iterator();
    boolean interrupted = false;
    try {
        do {
            try {
                Map<String, AttributeValue> item = it.next();
                DynamoDBEntryWithSize entryWithSize = new DynamoDBEntryWithSize(
                        item,
                        ItemSizeCalculator.calculateItemSizeInBytes(item));
                queue.put(entryWithSize);
            } catch (InterruptedException e) {
                interrupted = true;
                LOGGER.warn("interrupted when writing item to queue: "
                        + e.getMessage());
            }
        } while (it.hasNext());
    } finally {
        if (interrupted) {
            Thread.currentThread().interrupt();
        }
    }
    return null;
}
 

@Override
public void run() {
    System.out.println("Scanning " + tableName + " segment " + segment + " out of " + totalSegments + " segments " + itemLimit + " items at a time...");
    Map<String, AttributeValue> exclusiveStartKey = null;
    int totalScannedItemCount = 0;
    int totalScanRequestCount = 0;
    try {
        while(true) {
            ScanRequest scanRequest = new ScanRequest()
                .withTableName(tableName)
                .withLimit(itemLimit)
                .withExclusiveStartKey(exclusiveStartKey)
                .withTotalSegments(totalSegments)
                .withSegment(segment);
            
            ScanResult result = client.scan(scanRequest);
            
            totalScanRequestCount++;
            totalScannedItemCount += result.getScannedCount();
            
            // print items returned from scan request
            processScanResult(segment, result);
            
            exclusiveStartKey = result.getLastEvaluatedKey();
            if (exclusiveStartKey == null) {
                break;
            }
        }
    } catch (AmazonServiceException ase) {
        System.err.println(ase.getMessage());
    } finally {
        System.out.println("Scanned " + totalScannedItemCount + " items from segment " + segment + " out of " + totalSegments + " of " + tableName + " with " + totalScanRequestCount + " scan requests");
    }
}
 

public static int calculateScanResultSizeInBytes(ScanResult result) {
    final Iterator<Map<String, AttributeValue>> it = result.getItems().iterator();
    int totalBytes = 0;
    while(it.hasNext()){
        totalBytes += calculateItemSizeInBytes(it.next());
    }
    return totalBytes;
}
 

@Override
public SegmentedScanResult call() {
    ScanResult result = null;
    result = runWithBackoff();

    final ConsumedCapacity cc = result.getConsumedCapacity();

    if (cc != null && cc.getCapacityUnits() != null) {
        lastConsumedCapacity = result.getConsumedCapacity()
                .getCapacityUnits().intValue();
    } else if (result.getScannedCount() != null && result.getCount() != null) {

        final boolean isConsistent = request.getConsistentRead();
        int itemSize = isConsistent ? BootstrapConstants.STRONGLY_CONSISTENT_READ_ITEM_SIZE
                : BootstrapConstants.EVENTUALLY_CONSISTENT_READ_ITEM_SIZE;

        lastConsumedCapacity = (result.getScannedCount() / (int) Math.max(1.0, result.getCount()))
                * (ItemSizeCalculator.calculateScanResultSizeInBytes(result) / itemSize);
    }

    if (result.getLastEvaluatedKey() != null
            && !result.getLastEvaluatedKey().isEmpty()) {
        hasNext = true;
        request.setExclusiveStartKey(result.getLastEvaluatedKey());
    } else {
        hasNext = false;
    }

    if (lastConsumedCapacity > 0) {
        rateLimiter.acquire(lastConsumedCapacity);
    }
    return new SegmentedScanResult(result, request.getSegment());
}
 

@Override
public ScanContext call() throws Exception {
    try {
        final ScanRequest originalRequest = DynamoDbDelegate.copyScanRequest(request);
        final ScanResult result = next();
        return new ScanContext(originalRequest, result);
    } catch (BackendRuntimeException e) {
        throw e.getBackendException();
    }
}
 

private void testScanContainsItem(
        final TransactionManagerDynamoDBFacade facade,
        final Map<String, AttributeValue> item,
        final boolean filterAttributes) {
    ScanRequest scanRequest = new ScanRequest()
            .withTableName(INTEG_HASH_TABLE_NAME);
    if (filterAttributes) {
        scanRequest.setAttributesToGet(attributesToGet);
    }
    ScanResult scanResult = facade.scan(scanRequest);
    assertEquals(1, scanResult.getItems().size());
    assertContainsNoTransactionAttributes(scanResult.getItems().get(0));
    assertEquals(item, scanResult.getItems().get(0));
}
 

public void sweepForStuckAndOldTransactions() {
    print("\n*** sweepForStuckAndOldTransactions() ***\n");
    
    // The scan should be done in a loop to follow the LastEvaluatedKey, and done with following the best practices for scanning a table.
    // This includes sleeping between pages, using Limit to limit the throughput of each operation to avoid hotspots,
    // and using parallel scan.
    print("Scanning one full page of the transactions table");
    ScanResult result = dynamodb.scan(new ScanRequest()
        .withTableName(TX_TABLE_NAME));
    
    // Pick some duration where transactions should be rolled back if they were sitting there PENDING.
    // 
    //long rollbackAfterDurationMills = 5 * 60 * 1000; // Must be idle and PENDING for 5 minutes to be rolled back
    //long deleteAfterDurationMillis = 24 * 60 * 60 * 1000; // Must be completed for 24 hours to be deleted
    long rollbackAfterDurationMills = 1;
    long deleteAfterDurationMillis = 1;
    for(Map<String, AttributeValue> txItem : result.getItems()) {
        print("Sweeping transaction " + txItem);
        try {
            if(TransactionManager.isTransactionItem(txItem)) {
                Transaction t = txManager.resumeTransaction(txItem);    
                t.sweep(rollbackAfterDurationMills, deleteAfterDurationMillis);
                print("  - Swept transaction (but it might have been skipped)");
            }
        } catch (TransactionException e) {
            // Log and report an error "unsticking" this transaction, but keep going.
            print("  - Error sweeping transaction " + txItem + " " + e);
        }
    }
    
}
 

@Override
public ScanResult scan(String tableName, List<String> attributesToGet)
        throws AmazonServiceException, AmazonClientException {
    ScanRequest request = new ScanRequest()
            .withTableName(tableName)
            .withAttributesToGet(attributesToGet);
    return scan(request);
}
 

@Override
public ScanResult scan(String tableName, Map<String, Condition> scanFilter)
        throws AmazonServiceException, AmazonClientException {
    ScanRequest request = new ScanRequest()
            .withTableName(tableName)
            .withScanFilter(scanFilter);
    return scan(request);
}
 

/**
 * Test the getters and constructor of segmented scan result.
 */
@Test
public void test() {
    ScanResult result = new ScanResult();
    int numSegments = 3;
    SegmentedScanResult segmentedScanResult = new SegmentedScanResult(
            result, numSegments);

    assertSame(result, segmentedScanResult.getScanResult());
    assertEquals(numSegments, segmentedScanResult.getSegment());
}