com.amazonaws.services.dynamodbv2.document.BatchWriteItemOutcome Java Examples

The following examples show how to use com.amazonaws.services.dynamodbv2.document.BatchWriteItemOutcome. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: PutDynamoDBTest.java    From localization_nifi with Apache License 2.0 6 votes vote down vote up 
@Before
public void setUp() {
    outcome = new BatchWriteItemOutcome(result);
    result.setUnprocessedItems(new HashMap<String, List<WriteRequest>>());
    final DynamoDB mockDynamoDB = new DynamoDB(Regions.AP_NORTHEAST_1) {
        @Override
        public BatchWriteItemOutcome batchWriteItem(TableWriteItems... tableWriteItems) {
            return outcome;
        }
    };

    putDynamoDB = new PutDynamoDB() {
        @Override
        protected DynamoDB getDynamoDB() {
            return mockDynamoDB;
        }
    };

}
Example #2
Source File: AbstractWriteDynamoDBProcessor.java    From nifi with Apache License 2.0 6 votes vote down vote up 
/**
 * Helper method to handle unprocessed items items
 * @param session process session
 * @param keysToFlowFileMap map of flow db primary key to flow file
 * @param table dynamodb table
 * @param hashKeyName the hash key name
 * @param hashKeyValueType the hash key value
 * @param rangeKeyName the range key name
 * @param rangeKeyValueType range key value
 * @param outcome the write outcome
 */
protected void handleUnprocessedItems(final ProcessSession session, Map<ItemKeys, FlowFile> keysToFlowFileMap, final String table, final String hashKeyName, final String hashKeyValueType,
        final String rangeKeyName, final String rangeKeyValueType, BatchWriteItemOutcome outcome) {
    BatchWriteItemResult result = outcome.getBatchWriteItemResult();

    // Handle unprocessed items
    List<WriteRequest> unprocessedItems = result.getUnprocessedItems().get(table);
    if ( unprocessedItems != null && unprocessedItems.size() > 0 ) {
        for ( WriteRequest request : unprocessedItems) {
            Map<String,AttributeValue> item = getRequestItem(request);
            Object hashKeyValue = getValue(item, hashKeyName, hashKeyValueType);
            Object rangeKeyValue = getValue(item, rangeKeyName, rangeKeyValueType);

            sendUnprocessedToUnprocessedRelationship(session, keysToFlowFileMap, hashKeyValue, rangeKeyValue);
        }
    }
}
Example #3
Source File: DeleteDynamoDBTest.java    From nifi with Apache License 2.0 6 votes vote down vote up 
@Before
public void setUp() {
    outcome = new BatchWriteItemOutcome(result);
    result.setUnprocessedItems(new HashMap<String, List<WriteRequest>>());
    final DynamoDB mockDynamoDB = new DynamoDB(Regions.AP_NORTHEAST_1) {
        @Override
        public BatchWriteItemOutcome batchWriteItem(TableWriteItems... tableWriteItems) {
            return outcome;
        }
    };

    deleteDynamoDB = new DeleteDynamoDB() {
        @Override
        protected DynamoDB getDynamoDB() {
            return mockDynamoDB;
        }
    };

}
Example #4
Source File: PutDynamoDBTest.java    From nifi with Apache License 2.0 6 votes vote down vote up 
@Before
public void setUp() {
    outcome = new BatchWriteItemOutcome(result);
    result.setUnprocessedItems(new HashMap<String, List<WriteRequest>>());
    final DynamoDB mockDynamoDB = new DynamoDB(Regions.AP_NORTHEAST_1) {
        @Override
        public BatchWriteItemOutcome batchWriteItem(TableWriteItems... tableWriteItems) {
            return outcome;
        }
    };

    putDynamoDB = new PutDynamoDB() {
        @Override
        protected DynamoDB getDynamoDB() {
            return mockDynamoDB;
        }
    };

}
Example #5
Source File: DeleteDynamoDBTest.java    From localization_nifi with Apache License 2.0 6 votes vote down vote up 
@Before
public void setUp() {
    outcome = new BatchWriteItemOutcome(result);
    result.setUnprocessedItems(new HashMap<String, List<WriteRequest>>());
    final DynamoDB mockDynamoDB = new DynamoDB(Regions.AP_NORTHEAST_1) {
        @Override
        public BatchWriteItemOutcome batchWriteItem(TableWriteItems... tableWriteItems) {
            return outcome;
        }
    };

    deleteDynamoDB = new DeleteDynamoDB() {
        @Override
        protected DynamoDB getDynamoDB() {
            return mockDynamoDB;
        }
    };

}
Example #6
Source File: AbstractWriteDynamoDBProcessor.java    From localization_nifi with Apache License 2.0 6 votes vote down vote up 
/**
 * Helper method to handle unprocessed items items
 * @param session process session
 * @param keysToFlowFileMap map of flow db primary key to flow file
 * @param table dynamodb table
 * @param hashKeyName the hash key name
 * @param hashKeyValueType the hash key value
 * @param rangeKeyName the range key name
 * @param rangeKeyValueType range key value
 * @param outcome the write outcome
 */
protected void handleUnprocessedItems(final ProcessSession session, Map<ItemKeys, FlowFile> keysToFlowFileMap, final String table, final String hashKeyName, final String hashKeyValueType,
        final String rangeKeyName, final String rangeKeyValueType, BatchWriteItemOutcome outcome) {
    BatchWriteItemResult result = outcome.getBatchWriteItemResult();

    // Handle unprocessed items
    List<WriteRequest> unprocessedItems = result.getUnprocessedItems().get(table);
    if ( unprocessedItems != null && unprocessedItems.size() > 0 ) {
        for ( WriteRequest request : unprocessedItems) {
            Map<String,AttributeValue> item = getRequestItem(request);
            Object hashKeyValue = getValue(item, hashKeyName, hashKeyValueType);
            Object rangeKeyValue = getValue(item, rangeKeyName, rangeKeyValueType);

            sendUnprocessedToUnprocessedRelationship(session, keysToFlowFileMap, hashKeyValue, rangeKeyValue);
        }
    }
}
Example #7
Source File: DeleteDynamoDBTest.java    From localization_nifi with Apache License 2.0 5 votes vote down vote up 
@Test
public void testStringHashStringRangeDeleteThrowsServiceException() {
    final DynamoDB mockDynamoDB = new DynamoDB(Regions.AP_NORTHEAST_1) {
        @Override
        public BatchWriteItemOutcome batchWriteItem(TableWriteItems... tableWriteItems) {
            throw new AmazonServiceException("serviceException");
        }
    };

    deleteDynamoDB = new DeleteDynamoDB() {
        @Override
        protected DynamoDB getDynamoDB() {
            return mockDynamoDB;
        }
    };
    final TestRunner deleteRunner = TestRunners.newTestRunner(deleteDynamoDB);

    deleteRunner.setProperty(AbstractDynamoDBProcessor.ACCESS_KEY,"abcd");
    deleteRunner.setProperty(AbstractDynamoDBProcessor.SECRET_KEY, "cdef");
    deleteRunner.setProperty(AbstractDynamoDBProcessor.REGION, REGION);
    deleteRunner.setProperty(AbstractDynamoDBProcessor.TABLE, stringHashStringRangeTableName);
    deleteRunner.setProperty(AbstractDynamoDBProcessor.HASH_KEY_NAME, "hashS");
    deleteRunner.setProperty(AbstractDynamoDBProcessor.HASH_KEY_VALUE, "h1");
    deleteRunner.setProperty(AbstractDynamoDBProcessor.RANGE_KEY_NAME, "rangeS");
    deleteRunner.setProperty(AbstractDynamoDBProcessor.RANGE_KEY_VALUE, "r1");

    deleteRunner.enqueue(new byte[] {});

    deleteRunner.run(1);

    deleteRunner.assertAllFlowFilesTransferred(AbstractDynamoDBProcessor.REL_FAILURE, 1);
    List<MockFlowFile> flowFiles = deleteRunner.getFlowFilesForRelationship(AbstractDynamoDBProcessor.REL_FAILURE);
    for (MockFlowFile flowFile : flowFiles) {
        assertEquals("serviceException (Service: null; Status Code: 0; Error Code: null; Request ID: null)", flowFile.getAttribute(AbstractDynamoDBProcessor.DYNAMODB_ERROR_EXCEPTION_MESSAGE));
    }

}
Example #8
Source File: DynamoDBPersistenceService.java    From openhab1-addons with Eclipse Public License 2.0 5 votes vote down vote up 
@Override
public void run() {
    logger.debug("Error storing object to dynamo, unprocessed items: {}. Retrying with exponential back-off",
            unprocessedItems);
    lastException = null;
    while (!unprocessedItems.isEmpty() && retry < WAIT_MILLIS_IN_RETRIES.length) {
        if (!sleep()) {
            // Interrupted
            return;
        }
        retry++;
        try {
            BatchWriteItemOutcome outcome = DynamoDBPersistenceService.this.db.getDynamoDB()
                    .batchWriteItemUnprocessed(unprocessedItems);
            unprocessedItems = outcome.getUnprocessedItems();
            lastException = null;
        } catch (AmazonServiceException e) {
            if (e instanceof ResourceNotFoundException) {
                logger.debug(
                        "DynamoDB query raised unexpected exception: {}. This might happen if table was recently created",
                        e.getMessage());
            } else {
                logger.debug("DynamoDB query raised unexpected exception: {}.", e.getMessage());
            }
            lastException = e;
            continue;
        }
    }
    if (unprocessedItems.isEmpty()) {
        logger.debug("After {} retries successfully wrote all unprocessed items", retry);
    } else {
        logger.warn(
                "Even after retries failed to write some items. Last exception: {} {}, unprocessed items: {}",
                lastException == null ? "null" : lastException.getClass().getName(),
                lastException == null ? "null" : lastException.getMessage(), unprocessedItems);
    }
}
Example #9
Source File: DeleteDynamoDBTest.java    From nifi with Apache License 2.0 5 votes vote down vote up 
@Test
public void testStringHashStringRangeDeleteThrowsRuntimeException() {
    final DynamoDB mockDynamoDB = new DynamoDB(Regions.AP_NORTHEAST_1) {
        @Override
        public BatchWriteItemOutcome batchWriteItem(TableWriteItems... tableWriteItems) {
            throw new RuntimeException("runtimeException");
        }
    };

    deleteDynamoDB = new DeleteDynamoDB() {
        @Override
        protected DynamoDB getDynamoDB() {
            return mockDynamoDB;
        }
    };
    final TestRunner deleteRunner = TestRunners.newTestRunner(deleteDynamoDB);

    deleteRunner.setProperty(AbstractDynamoDBProcessor.ACCESS_KEY,"abcd");
    deleteRunner.setProperty(AbstractDynamoDBProcessor.SECRET_KEY, "cdef");
    deleteRunner.setProperty(AbstractDynamoDBProcessor.REGION, REGION);
    deleteRunner.setProperty(AbstractDynamoDBProcessor.TABLE, stringHashStringRangeTableName);
    deleteRunner.setProperty(AbstractDynamoDBProcessor.HASH_KEY_NAME, "hashS");
    deleteRunner.setProperty(AbstractDynamoDBProcessor.HASH_KEY_VALUE, "h1");
    deleteRunner.setProperty(AbstractDynamoDBProcessor.RANGE_KEY_NAME, "rangeS");
    deleteRunner.setProperty(AbstractDynamoDBProcessor.RANGE_KEY_VALUE, "r1");

    deleteRunner.enqueue(new byte[] {});

    deleteRunner.run(1);

    deleteRunner.assertAllFlowFilesTransferred(AbstractDynamoDBProcessor.REL_FAILURE, 1);
    List<MockFlowFile> flowFiles = deleteRunner.getFlowFilesForRelationship(AbstractDynamoDBProcessor.REL_FAILURE);
    for (MockFlowFile flowFile : flowFiles) {
        assertEquals("runtimeException", flowFile.getAttribute(AbstractDynamoDBProcessor.DYNAMODB_ERROR_EXCEPTION_MESSAGE));
    }

}
Example #10
Source File: DeleteDynamoDBTest.java    From nifi with Apache License 2.0 5 votes vote down vote up 
@Test
public void testStringHashStringRangeDeleteThrowsClientException() {
    final DynamoDB mockDynamoDB = new DynamoDB(Regions.AP_NORTHEAST_1) {
        @Override
        public BatchWriteItemOutcome batchWriteItem(TableWriteItems... tableWriteItems) {
            throw new AmazonClientException("clientException");
        }
    };

    deleteDynamoDB = new DeleteDynamoDB() {
        @Override
        protected DynamoDB getDynamoDB() {
            return mockDynamoDB;
        }
    };
    final TestRunner deleteRunner = TestRunners.newTestRunner(deleteDynamoDB);

    deleteRunner.setProperty(AbstractDynamoDBProcessor.ACCESS_KEY,"abcd");
    deleteRunner.setProperty(AbstractDynamoDBProcessor.SECRET_KEY, "cdef");
    deleteRunner.setProperty(AbstractDynamoDBProcessor.REGION, REGION);
    deleteRunner.setProperty(AbstractDynamoDBProcessor.TABLE, stringHashStringRangeTableName);
    deleteRunner.setProperty(AbstractDynamoDBProcessor.HASH_KEY_NAME, "hashS");
    deleteRunner.setProperty(AbstractDynamoDBProcessor.HASH_KEY_VALUE, "h1");
    deleteRunner.setProperty(AbstractDynamoDBProcessor.RANGE_KEY_NAME, "rangeS");
    deleteRunner.setProperty(AbstractDynamoDBProcessor.RANGE_KEY_VALUE, "r1");

    deleteRunner.enqueue(new byte[] {});

    deleteRunner.run(1);

    deleteRunner.assertAllFlowFilesTransferred(AbstractDynamoDBProcessor.REL_FAILURE, 1);
    List<MockFlowFile> flowFiles = deleteRunner.getFlowFilesForRelationship(AbstractDynamoDBProcessor.REL_FAILURE);
    for (MockFlowFile flowFile : flowFiles) {
        assertEquals("clientException", flowFile.getAttribute(AbstractDynamoDBProcessor.DYNAMODB_ERROR_EXCEPTION_MESSAGE));
    }

}
Example #11
Source File: DeleteDynamoDBTest.java    From nifi with Apache License 2.0 5 votes vote down vote up 
@Test
public void testStringHashStringRangeDeleteThrowsServiceException() {
    final DynamoDB mockDynamoDB = new DynamoDB(Regions.AP_NORTHEAST_1) {
        @Override
        public BatchWriteItemOutcome batchWriteItem(TableWriteItems... tableWriteItems) {
            throw new AmazonServiceException("serviceException");
        }
    };

    deleteDynamoDB = new DeleteDynamoDB() {
        @Override
        protected DynamoDB getDynamoDB() {
            return mockDynamoDB;
        }
    };
    final TestRunner deleteRunner = TestRunners.newTestRunner(deleteDynamoDB);

    deleteRunner.setProperty(AbstractDynamoDBProcessor.ACCESS_KEY,"abcd");
    deleteRunner.setProperty(AbstractDynamoDBProcessor.SECRET_KEY, "cdef");
    deleteRunner.setProperty(AbstractDynamoDBProcessor.REGION, REGION);
    deleteRunner.setProperty(AbstractDynamoDBProcessor.TABLE, stringHashStringRangeTableName);
    deleteRunner.setProperty(AbstractDynamoDBProcessor.HASH_KEY_NAME, "hashS");
    deleteRunner.setProperty(AbstractDynamoDBProcessor.HASH_KEY_VALUE, "h1");
    deleteRunner.setProperty(AbstractDynamoDBProcessor.RANGE_KEY_NAME, "rangeS");
    deleteRunner.setProperty(AbstractDynamoDBProcessor.RANGE_KEY_VALUE, "r1");

    deleteRunner.enqueue(new byte[] {});

    deleteRunner.run(1);

    deleteRunner.assertAllFlowFilesTransferred(AbstractDynamoDBProcessor.REL_FAILURE, 1);
    List<MockFlowFile> flowFiles = deleteRunner.getFlowFilesForRelationship(AbstractDynamoDBProcessor.REL_FAILURE);
    for (MockFlowFile flowFile : flowFiles) {
        assertEquals("serviceException (Service: null; Status Code: 0; Error Code: null; Request ID: null)", flowFile.getAttribute(AbstractDynamoDBProcessor.DYNAMODB_ERROR_EXCEPTION_MESSAGE));
    }

}
Example #12
Source File: PutDynamoDBTest.java    From localization_nifi with Apache License 2.0 5 votes vote down vote up 
@Test
public void testStringHashStringRangePutThrowsServiceException() {
    final DynamoDB mockDynamoDB = new DynamoDB(Regions.AP_NORTHEAST_1) {
        @Override
        public BatchWriteItemOutcome batchWriteItem(TableWriteItems... tableWriteItems) {
            throw new AmazonServiceException("serviceException");
        }
    };

    putDynamoDB = new PutDynamoDB() {
        @Override
        protected DynamoDB getDynamoDB() {
            return mockDynamoDB;
        }
    };
    final TestRunner putRunner = TestRunners.newTestRunner(putDynamoDB);

    putRunner.setProperty(AbstractDynamoDBProcessor.ACCESS_KEY,"abcd");
    putRunner.setProperty(AbstractDynamoDBProcessor.SECRET_KEY, "cdef");
    putRunner.setProperty(AbstractDynamoDBProcessor.REGION, REGION);
    putRunner.setProperty(AbstractDynamoDBProcessor.TABLE, stringHashStringRangeTableName);
    putRunner.setProperty(AbstractDynamoDBProcessor.HASH_KEY_NAME, "hashS");
    putRunner.setProperty(AbstractDynamoDBProcessor.HASH_KEY_VALUE, "h1");
    putRunner.setProperty(AbstractDynamoDBProcessor.RANGE_KEY_NAME, "rangeS");
    putRunner.setProperty(AbstractDynamoDBProcessor.RANGE_KEY_VALUE, "r1");
    putRunner.setProperty(AbstractWriteDynamoDBProcessor.JSON_DOCUMENT, "document");
    String document = "{\"name\":\"john\"}";
    putRunner.enqueue(document.getBytes());

    putRunner.run(1);

    putRunner.assertAllFlowFilesTransferred(AbstractDynamoDBProcessor.REL_FAILURE, 1);
    List<MockFlowFile> flowFiles = putRunner.getFlowFilesForRelationship(AbstractDynamoDBProcessor.REL_FAILURE);
    for (MockFlowFile flowFile : flowFiles) {
        assertEquals("serviceException (Service: null; Status Code: 0; Error Code: null; Request ID: null)", flowFile.getAttribute(AbstractDynamoDBProcessor.DYNAMODB_ERROR_EXCEPTION_MESSAGE));
    }

}
Example #13
Source File: PutDynamoDBTest.java    From nifi with Apache License 2.0 5 votes vote down vote up 
@Test
public void testStringHashStringRangePutThrowsRuntimeException() {
    final DynamoDB mockDynamoDB = new DynamoDB(Regions.AP_NORTHEAST_1) {
        @Override
        public BatchWriteItemOutcome batchWriteItem(TableWriteItems... tableWriteItems) {
            throw new RuntimeException("runtimeException");
        }
    };

    putDynamoDB = new PutDynamoDB() {
        @Override
        protected DynamoDB getDynamoDB() {
            return mockDynamoDB;
        }
    };
    final TestRunner putRunner = TestRunners.newTestRunner(putDynamoDB);

    putRunner.setProperty(AbstractDynamoDBProcessor.ACCESS_KEY,"abcd");
    putRunner.setProperty(AbstractDynamoDBProcessor.SECRET_KEY, "cdef");
    putRunner.setProperty(AbstractDynamoDBProcessor.REGION, REGION);
    putRunner.setProperty(AbstractDynamoDBProcessor.TABLE, stringHashStringRangeTableName);
    putRunner.setProperty(AbstractDynamoDBProcessor.HASH_KEY_NAME, "hashS");
    putRunner.setProperty(AbstractDynamoDBProcessor.HASH_KEY_VALUE, "h1");
    putRunner.setProperty(AbstractDynamoDBProcessor.RANGE_KEY_NAME, "rangeS");
    putRunner.setProperty(AbstractDynamoDBProcessor.RANGE_KEY_VALUE, "r1");
    putRunner.setProperty(AbstractWriteDynamoDBProcessor.JSON_DOCUMENT, "document");
    String document = "{\"name\":\"john\"}";
    putRunner.enqueue(document.getBytes());

    putRunner.run(1);

    putRunner.assertAllFlowFilesTransferred(AbstractDynamoDBProcessor.REL_FAILURE, 1);
    List<MockFlowFile> flowFiles = putRunner.getFlowFilesForRelationship(AbstractDynamoDBProcessor.REL_FAILURE);
    for (MockFlowFile flowFile : flowFiles) {
        assertEquals("runtimeException", flowFile.getAttribute(AbstractDynamoDBProcessor.DYNAMODB_ERROR_EXCEPTION_MESSAGE));
    }

}
Example #14
Source File: PutDynamoDBTest.java    From nifi with Apache License 2.0 5 votes vote down vote up 
@Test
public void testStringHashStringRangePutThrowsClientException() {
    final DynamoDB mockDynamoDB = new DynamoDB(Regions.AP_NORTHEAST_1) {
        @Override
        public BatchWriteItemOutcome batchWriteItem(TableWriteItems... tableWriteItems) {
            throw new AmazonClientException("clientException");
        }
    };

    putDynamoDB = new PutDynamoDB() {
        @Override
        protected DynamoDB getDynamoDB() {
            return mockDynamoDB;
        }
    };
    final TestRunner putRunner = TestRunners.newTestRunner(putDynamoDB);

    putRunner.setProperty(AbstractDynamoDBProcessor.ACCESS_KEY,"abcd");
    putRunner.setProperty(AbstractDynamoDBProcessor.SECRET_KEY, "cdef");
    putRunner.setProperty(AbstractDynamoDBProcessor.REGION, REGION);
    putRunner.setProperty(AbstractDynamoDBProcessor.TABLE, stringHashStringRangeTableName);
    putRunner.setProperty(AbstractDynamoDBProcessor.HASH_KEY_NAME, "hashS");
    putRunner.setProperty(AbstractDynamoDBProcessor.HASH_KEY_VALUE, "h1");
    putRunner.setProperty(AbstractDynamoDBProcessor.RANGE_KEY_NAME, "rangeS");
    putRunner.setProperty(AbstractDynamoDBProcessor.RANGE_KEY_VALUE, "r1");
    putRunner.setProperty(AbstractWriteDynamoDBProcessor.JSON_DOCUMENT, "document");
    String document = "{\"name\":\"john\"}";
    putRunner.enqueue(document.getBytes());

    putRunner.run(1);

    putRunner.assertAllFlowFilesTransferred(AbstractDynamoDBProcessor.REL_FAILURE, 1);
    List<MockFlowFile> flowFiles = putRunner.getFlowFilesForRelationship(AbstractDynamoDBProcessor.REL_FAILURE);
    for (MockFlowFile flowFile : flowFiles) {
        assertEquals("clientException", flowFile.getAttribute(AbstractDynamoDBProcessor.DYNAMODB_ERROR_EXCEPTION_MESSAGE));
    }

}
Example #15
Source File: PutDynamoDBTest.java    From nifi with Apache License 2.0 5 votes vote down vote up 
@Test
public void testStringHashStringRangePutThrowsServiceException() {
    final DynamoDB mockDynamoDB = new DynamoDB(Regions.AP_NORTHEAST_1) {
        @Override
        public BatchWriteItemOutcome batchWriteItem(TableWriteItems... tableWriteItems) {
            throw new AmazonServiceException("serviceException");
        }
    };

    putDynamoDB = new PutDynamoDB() {
        @Override
        protected DynamoDB getDynamoDB() {
            return mockDynamoDB;
        }
    };
    final TestRunner putRunner = TestRunners.newTestRunner(putDynamoDB);

    putRunner.setProperty(AbstractDynamoDBProcessor.ACCESS_KEY,"abcd");
    putRunner.setProperty(AbstractDynamoDBProcessor.SECRET_KEY, "cdef");
    putRunner.setProperty(AbstractDynamoDBProcessor.REGION, REGION);
    putRunner.setProperty(AbstractDynamoDBProcessor.TABLE, stringHashStringRangeTableName);
    putRunner.setProperty(AbstractDynamoDBProcessor.HASH_KEY_NAME, "hashS");
    putRunner.setProperty(AbstractDynamoDBProcessor.HASH_KEY_VALUE, "h1");
    putRunner.setProperty(AbstractDynamoDBProcessor.RANGE_KEY_NAME, "rangeS");
    putRunner.setProperty(AbstractDynamoDBProcessor.RANGE_KEY_VALUE, "r1");
    putRunner.setProperty(AbstractWriteDynamoDBProcessor.JSON_DOCUMENT, "document");
    String document = "{\"name\":\"john\"}";
    putRunner.enqueue(document.getBytes());

    putRunner.run(1);

    putRunner.assertAllFlowFilesTransferred(AbstractDynamoDBProcessor.REL_FAILURE, 1);
    List<MockFlowFile> flowFiles = putRunner.getFlowFilesForRelationship(AbstractDynamoDBProcessor.REL_FAILURE);
    for (MockFlowFile flowFile : flowFiles) {
        assertEquals("serviceException (Service: null; Status Code: 0; Error Code: null; Request ID: null)", flowFile.getAttribute(AbstractDynamoDBProcessor.DYNAMODB_ERROR_EXCEPTION_MESSAGE));
    }

}
Example #16
Source File: PutDynamoDBTest.java    From localization_nifi with Apache License 2.0 5 votes vote down vote up 
@Test
public void testStringHashStringRangePutThrowsClientException() {
    final DynamoDB mockDynamoDB = new DynamoDB(Regions.AP_NORTHEAST_1) {
        @Override
        public BatchWriteItemOutcome batchWriteItem(TableWriteItems... tableWriteItems) {
            throw new AmazonClientException("clientException");
        }
    };

    putDynamoDB = new PutDynamoDB() {
        @Override
        protected DynamoDB getDynamoDB() {
            return mockDynamoDB;
        }
    };
    final TestRunner putRunner = TestRunners.newTestRunner(putDynamoDB);

    putRunner.setProperty(AbstractDynamoDBProcessor.ACCESS_KEY,"abcd");
    putRunner.setProperty(AbstractDynamoDBProcessor.SECRET_KEY, "cdef");
    putRunner.setProperty(AbstractDynamoDBProcessor.REGION, REGION);
    putRunner.setProperty(AbstractDynamoDBProcessor.TABLE, stringHashStringRangeTableName);
    putRunner.setProperty(AbstractDynamoDBProcessor.HASH_KEY_NAME, "hashS");
    putRunner.setProperty(AbstractDynamoDBProcessor.HASH_KEY_VALUE, "h1");
    putRunner.setProperty(AbstractDynamoDBProcessor.RANGE_KEY_NAME, "rangeS");
    putRunner.setProperty(AbstractDynamoDBProcessor.RANGE_KEY_VALUE, "r1");
    putRunner.setProperty(AbstractWriteDynamoDBProcessor.JSON_DOCUMENT, "document");
    String document = "{\"name\":\"john\"}";
    putRunner.enqueue(document.getBytes());

    putRunner.run(1);

    putRunner.assertAllFlowFilesTransferred(AbstractDynamoDBProcessor.REL_FAILURE, 1);
    List<MockFlowFile> flowFiles = putRunner.getFlowFilesForRelationship(AbstractDynamoDBProcessor.REL_FAILURE);
    for (MockFlowFile flowFile : flowFiles) {
        assertEquals("clientException", flowFile.getAttribute(AbstractDynamoDBProcessor.DYNAMODB_ERROR_EXCEPTION_MESSAGE));
    }

}
Example #17
Source File: DynamoDBBackendImpl.java    From fiware-cygnus with GNU Affero General Public License v3.0 5 votes vote down vote up 
@Override
public void putItems(String tableName, ArrayList<Item> aggregation) throws Exception {
    try {
        TableWriteItems tableWriteItems = new TableWriteItems(tableName);
        tableWriteItems.withItemsToPut(aggregation);
        BatchWriteItemOutcome outcome = dynamoDB.batchWriteItem(tableWriteItems);
    } catch (Exception e) {
        LOGGER.error("Error while putting a batch of items in the table " + tableName
                + ". Details=" + e.getMessage());
    } // try catch
}
Example #18
Source File: DeleteDynamoDBTest.java    From localization_nifi with Apache License 2.0 5 votes vote down vote up 
@Test
public void testStringHashStringRangeDeleteThrowsRuntimeException() {
    final DynamoDB mockDynamoDB = new DynamoDB(Regions.AP_NORTHEAST_1) {
        @Override
        public BatchWriteItemOutcome batchWriteItem(TableWriteItems... tableWriteItems) {
            throw new RuntimeException("runtimeException");
        }
    };

    deleteDynamoDB = new DeleteDynamoDB() {
        @Override
        protected DynamoDB getDynamoDB() {
            return mockDynamoDB;
        }
    };
    final TestRunner deleteRunner = TestRunners.newTestRunner(deleteDynamoDB);

    deleteRunner.setProperty(AbstractDynamoDBProcessor.ACCESS_KEY,"abcd");
    deleteRunner.setProperty(AbstractDynamoDBProcessor.SECRET_KEY, "cdef");
    deleteRunner.setProperty(AbstractDynamoDBProcessor.REGION, REGION);
    deleteRunner.setProperty(AbstractDynamoDBProcessor.TABLE, stringHashStringRangeTableName);
    deleteRunner.setProperty(AbstractDynamoDBProcessor.HASH_KEY_NAME, "hashS");
    deleteRunner.setProperty(AbstractDynamoDBProcessor.HASH_KEY_VALUE, "h1");
    deleteRunner.setProperty(AbstractDynamoDBProcessor.RANGE_KEY_NAME, "rangeS");
    deleteRunner.setProperty(AbstractDynamoDBProcessor.RANGE_KEY_VALUE, "r1");

    deleteRunner.enqueue(new byte[] {});

    deleteRunner.run(1);

    deleteRunner.assertAllFlowFilesTransferred(AbstractDynamoDBProcessor.REL_FAILURE, 1);
    List<MockFlowFile> flowFiles = deleteRunner.getFlowFilesForRelationship(AbstractDynamoDBProcessor.REL_FAILURE);
    for (MockFlowFile flowFile : flowFiles) {
        assertEquals("runtimeException", flowFile.getAttribute(AbstractDynamoDBProcessor.DYNAMODB_ERROR_EXCEPTION_MESSAGE));
    }

}
Example #19
Source File: DeleteDynamoDBTest.java    From localization_nifi with Apache License 2.0 5 votes vote down vote up 
@Test
public void testStringHashStringRangeDeleteThrowsClientException() {
    final DynamoDB mockDynamoDB = new DynamoDB(Regions.AP_NORTHEAST_1) {
        @Override
        public BatchWriteItemOutcome batchWriteItem(TableWriteItems... tableWriteItems) {
            throw new AmazonClientException("clientException");
        }
    };

    deleteDynamoDB = new DeleteDynamoDB() {
        @Override
        protected DynamoDB getDynamoDB() {
            return mockDynamoDB;
        }
    };
    final TestRunner deleteRunner = TestRunners.newTestRunner(deleteDynamoDB);

    deleteRunner.setProperty(AbstractDynamoDBProcessor.ACCESS_KEY,"abcd");
    deleteRunner.setProperty(AbstractDynamoDBProcessor.SECRET_KEY, "cdef");
    deleteRunner.setProperty(AbstractDynamoDBProcessor.REGION, REGION);
    deleteRunner.setProperty(AbstractDynamoDBProcessor.TABLE, stringHashStringRangeTableName);
    deleteRunner.setProperty(AbstractDynamoDBProcessor.HASH_KEY_NAME, "hashS");
    deleteRunner.setProperty(AbstractDynamoDBProcessor.HASH_KEY_VALUE, "h1");
    deleteRunner.setProperty(AbstractDynamoDBProcessor.RANGE_KEY_NAME, "rangeS");
    deleteRunner.setProperty(AbstractDynamoDBProcessor.RANGE_KEY_VALUE, "r1");

    deleteRunner.enqueue(new byte[] {});

    deleteRunner.run(1);

    deleteRunner.assertAllFlowFilesTransferred(AbstractDynamoDBProcessor.REL_FAILURE, 1);
    List<MockFlowFile> flowFiles = deleteRunner.getFlowFilesForRelationship(AbstractDynamoDBProcessor.REL_FAILURE);
    for (MockFlowFile flowFile : flowFiles) {
        assertEquals("clientException", flowFile.getAttribute(AbstractDynamoDBProcessor.DYNAMODB_ERROR_EXCEPTION_MESSAGE));
    }

}
Example #20
Source File: PutDynamoDBTest.java    From localization_nifi with Apache License 2.0 5 votes vote down vote up 
@Test
public void testStringHashStringRangePutThrowsRuntimeException() {
    final DynamoDB mockDynamoDB = new DynamoDB(Regions.AP_NORTHEAST_1) {
        @Override
        public BatchWriteItemOutcome batchWriteItem(TableWriteItems... tableWriteItems) {
            throw new RuntimeException("runtimeException");
        }
    };

    putDynamoDB = new PutDynamoDB() {
        @Override
        protected DynamoDB getDynamoDB() {
            return mockDynamoDB;
        }
    };
    final TestRunner putRunner = TestRunners.newTestRunner(putDynamoDB);

    putRunner.setProperty(AbstractDynamoDBProcessor.ACCESS_KEY,"abcd");
    putRunner.setProperty(AbstractDynamoDBProcessor.SECRET_KEY, "cdef");
    putRunner.setProperty(AbstractDynamoDBProcessor.REGION, REGION);
    putRunner.setProperty(AbstractDynamoDBProcessor.TABLE, stringHashStringRangeTableName);
    putRunner.setProperty(AbstractDynamoDBProcessor.HASH_KEY_NAME, "hashS");
    putRunner.setProperty(AbstractDynamoDBProcessor.HASH_KEY_VALUE, "h1");
    putRunner.setProperty(AbstractDynamoDBProcessor.RANGE_KEY_NAME, "rangeS");
    putRunner.setProperty(AbstractDynamoDBProcessor.RANGE_KEY_VALUE, "r1");
    putRunner.setProperty(AbstractWriteDynamoDBProcessor.JSON_DOCUMENT, "document");
    String document = "{\"name\":\"john\"}";
    putRunner.enqueue(document.getBytes());

    putRunner.run(1);

    putRunner.assertAllFlowFilesTransferred(AbstractDynamoDBProcessor.REL_FAILURE, 1);
    List<MockFlowFile> flowFiles = putRunner.getFlowFilesForRelationship(AbstractDynamoDBProcessor.REL_FAILURE);
    for (MockFlowFile flowFile : flowFiles) {
        assertEquals("runtimeException", flowFile.getAttribute(AbstractDynamoDBProcessor.DYNAMODB_ERROR_EXCEPTION_MESSAGE));
    }

}
Example #21
Source File: DocumentAPIBatchWrite.java    From aws-doc-sdk-examples with Apache License 2.0 4 votes vote down vote up 
private static void writeMultipleItemsBatchWrite() {
    try {

        // Add a new item to Forum
        TableWriteItems forumTableWriteItems = new TableWriteItems(forumTableName) // Forum
            .withItemsToPut(new Item().withPrimaryKey("Name", "Amazon RDS").withNumber("Threads", 0));

        // Add a new item, and delete an existing item, from Thread
        // This table has a partition key and range key, so need to specify
        // both of them
        TableWriteItems threadTableWriteItems = new TableWriteItems(threadTableName)
            .withItemsToPut(new Item().withPrimaryKey("ForumName", "Amazon RDS", "Subject", "Amazon RDS Thread 1")
                .withString("Message", "ElastiCache Thread 1 message")
                .withStringSet("Tags", new HashSet<String>(Arrays.asList("cache", "in-memory"))))
            .withHashAndRangeKeysToDelete("ForumName", "Subject", "Amazon S3", "S3 Thread 100");

        System.out.println("Making the request.");
        BatchWriteItemOutcome outcome = dynamoDB.batchWriteItem(forumTableWriteItems, threadTableWriteItems);

        do {

            // Check for unprocessed keys which could happen if you exceed
            // provisioned throughput

            Map<String, List<WriteRequest>> unprocessedItems = outcome.getUnprocessedItems();

            if (outcome.getUnprocessedItems().size() == 0) {
                System.out.println("No unprocessed items found");
            }
            else {
                System.out.println("Retrieving the unprocessed items");
                outcome = dynamoDB.batchWriteItemUnprocessed(unprocessedItems);
            }

        } while (outcome.getUnprocessedItems().size() > 0);

    }
    catch (Exception e) {
        System.err.println("Failed to retrieve items: ");
        e.printStackTrace(System.err);
    }

}
Example #22
Source File: DocumentAPIBatchWrite.java    From aws-dynamodb-examples with Apache License 2.0 4 votes vote down vote up 
private static void writeMultipleItemsBatchWrite() {
    try {                    

        // Add a new item to Forum
        TableWriteItems forumTableWriteItems = new TableWriteItems(forumTableName) //Forum
            .withItemsToPut(new Item()
                .withPrimaryKey("Name", "Amazon RDS")
                .withNumber("Threads", 0));

        // Add a new item, and delete an existing item, from Thread
        TableWriteItems threadTableWriteItems = new TableWriteItems(threadTableName)
        .withItemsToPut(new Item()
            .withPrimaryKey("ForumName","Amazon RDS","Subject","Amazon RDS Thread 1")
            .withString("Message", "ElasticCache Thread 1 message")
            .withStringSet("Tags", new HashSet<String>(
                Arrays.asList("cache", "in-memory"))))
        .withHashAndRangeKeysToDelete("ForumName","Subject", "Amazon S3", "S3 Thread 100");

        System.out.println("Making the request.");
        BatchWriteItemOutcome outcome = dynamoDB.batchWriteItem(forumTableWriteItems, threadTableWriteItems);

        do {

            // Check for unprocessed keys which could happen if you exceed provisioned throughput

            Map<String, List<WriteRequest>> unprocessedItems = outcome.getUnprocessedItems();

            if (outcome.getUnprocessedItems().size() == 0) {
                System.out.println("No unprocessed items found");
            } else {
                System.out.println("Retrieving the unprocessed items");
                outcome = dynamoDB.batchWriteItemUnprocessed(unprocessedItems);
            }

        } while (outcome.getUnprocessedItems().size() > 0);

    }  catch (Exception e) {
        System.err.println("Failed to retrieve items: ");
        e.printStackTrace(System.err);
    }  

}