com.amazonaws.services.sqs.AmazonSQSClient Java Examples

private void initSqs(String keyId, String keySecret) {
    log.debug("Initializing SQS Client");
    sqs = new AmazonSQSClient(new BasicAWSCredentials(keyId, keySecret));
    
    //SQS Consistency Queue
    consistencyQueue = conf.get("s3mper.alert.sqs.queue", consistencyQueue);
    consistencyQueueUrl = sqs.getQueueUrl(new GetQueueUrlRequest(consistencyQueue)).getQueueUrl();
    
    //SQS Timeout Queue
    timeoutQueue = conf.get("s3mper.timeout.sqs.queue", timeoutQueue);
    timeoutQueueUrl = sqs.getQueueUrl(new GetQueueUrlRequest(timeoutQueue)).getQueueUrl();
    
    //SQS Notification Queue
    notificationQueue = conf.get("s3mper.notification.sqs.queue", notificationQueue);
    notificationQueueUrl = sqs.getQueueUrl(new GetQueueUrlRequest(notificationQueue)).getQueueUrl();
    
    //Disable reporting  (Testing purposes mostly)
    reportingDisabled = conf.getBoolean("s3mper.reporting.disabled", reportingDisabled);
}
 

public void initalize(URI uri, Configuration conf) {
    this.conf = conf;
    
    String keyId = conf.get("fs."+uri.getScheme()+".awsAccessKeyId");
    String keySecret = conf.get("fs."+uri.getScheme()+".awsSecretAccessKey");
    
    //An override option for accessing across accounts
    keyId = conf.get("fs."+uri.getScheme()+".override.awsAccessKeyId", keyId);
    keySecret = conf.get("fs."+uri.getScheme()+".override.awsSecretAccessKey", keySecret);
    
    sqs = new AmazonSQSClient(new BasicAWSCredentials(keyId, keySecret));
    
    //SQS Consistency Queue
    consistencyQueue = conf.get("fs"+uri.getScheme()+".alert.sqs.queue", consistencyQueue);
    consistencyQueue = sqs.getQueueUrl(new GetQueueUrlRequest(consistencyQueue)).getQueueUrl();
    
    //SQS Timeout Queue
    timeoutQueue = conf.get("fs"+uri.getScheme()+".timeout.sqs.queue", timeoutQueue);
    timeoutQueue = sqs.getQueueUrl(new GetQueueUrlRequest(timeoutQueue)).getQueueUrl();
}
 

public static String getQueueArnByUrl( final AmazonSQSClient sqs, final String queueUrl ) throws Exception {

        try {
            GetQueueAttributesRequest queueAttributesRequest =
                new GetQueueAttributesRequest( queueUrl ).withAttributeNames( "All" );

            GetQueueAttributesResult queueAttributesResult = sqs.getQueueAttributes( queueAttributesRequest );
            Map<String, String> sqsAttributeMap = queueAttributesResult.getAttributes();

            return sqsAttributeMap.get( "QueueArn" );
        }
        catch ( Exception e ) {
            logger.error( "Failed to get queue URL from service", e );
            throw e;
        }
    }
 

/********************** Member Functions **************************/

	public AwsGlacierInventoryRetriever(String region) {
		// Get credentials from credentials file, environment variable, or 
		// Java property. 
		// See http://docs.aws.amazon.com/AWSJavaSDK/latest/javadoc/com/amazonaws/auth/DefaultAWSCredentialsProviderChain.html
		AWSCredentialsProviderChain credentialsProvider = 
				new DefaultAWSCredentialsProviderChain();
		AWSCredentials credentials = credentialsProvider.getCredentials();
		logger.debug("Read in credentials AWSAccessKeyId={} AWSSecretKey={}...",
				credentials.getAWSAccessKeyId(), 
				credentials.getAWSSecretKey().substring(0, 4));
		
		// Create the glacier client and set to specified region.
		glacierClient = new AmazonGlacierClient(credentials);
		glacierClient.setEndpoint("https://glacier." + region + ".amazonaws.com");
		
		// Set up params needed for retrieving vault inventory
        sqsClient = new AmazonSQSClient(credentials);
        sqsClient.setEndpoint("https://sqs." + region + ".amazonaws.com");
        snsClient = new AmazonSNSClient(credentials);
        snsClient.setEndpoint("https://sns." + region + ".amazonaws.com");
        setupSQS();
        setupSNS();
	}
 

public TOCQueue(boolean isConsumer, String awsAccessKey, String awsSecretKey, String sqsQueueName, TOCPayloadHandler tocPayloadHandler) throws Exception {
	super();

	mySourceIdentifier = determineHostName() + "-" + UUID.randomUUID().toString().replace("-", "").substring(0,4);
	this.sqsQueueName = sqsQueueName;
	this.tocPayloadHandler = tocPayloadHandler;
	
	if (!isConsumer) { // then I am the master...
		canDestroyQueue = true; 
		this.sqsQueueName += "-" + mySourceIdentifier;
	}
	

	sqsClient = new AmazonSQSClient(new BasicAWSCredentials(awsAccessKey, awsSecretKey));

	
	connectToQueue(isConsumer, 1000);
	
	if (isConsumer) {
		this.consumerThread = new Thread(this,"TOCQueue["+myId+"] msg consumer thread");
	}
	
	logger.info("\n-------------------------------------------\n" +
				"TOC Queue["+myId+"]: ALL SQS resources hooked up OK: "+this.tocQueueUrl+"\n" +
				"-------------------------------------------\n");
}
 

private SQSConnectionFactory(final Builder builder) {
    this.providerConfiguration = builder.providerConfiguration;
    this.amazonSQSClientSupplier = new AmazonSQSClientSupplier() {
        @Override
        public AmazonSQS get() {
            AmazonSQSClient amazonSQSClient = new AmazonSQSClient(builder.awsCredentialsProvider, builder.clientConfiguration);
            if (builder.region != null) {
                amazonSQSClient.setRegion(builder.region);
            }
            if (builder.endpoint != null) {
                amazonSQSClient.setEndpoint(builder.endpoint);
            }
            if (builder.signerRegionOverride != null) {
                amazonSQSClient.setSignerRegionOverride(builder.signerRegionOverride);
            }
            return amazonSQSClient;
        }
    };
}
 

@Before
public void setup() {
    injector = Guice.createInjector(
            new SuroPlugin() {
                @Override
                protected void configure() {
                    this.addSinkType("TestSink", TestSinkManager.TestSink.class);

                    this.addNoticeType(NoNotice.TYPE, NoNotice.class);
                    this.addNoticeType(QueueNotice.TYPE, QueueNotice.class);
                    this.addNoticeType(SQSNotice.TYPE, SQSNotice.class);
                }
            },
            new AbstractModule() {
                @Override
                protected void configure() {
                    bind(ObjectMapper.class).to(DefaultObjectMapper.class);
                    bind(AWSCredentialsProvider.class).to(PropertyAWSCredentialsProvider.class);

                    bind(AmazonSQSClient.class).toProvider(AmazonSQSClientProvider.class).asEagerSingleton();
                }
            }
    );
}
 

public ControlChannel(boolean callerIsMaster,
					  String awsAccessKey, String awsSecretKey, String snsControlTopicName, 
					  String userAccountPrincipalId, 
					  String userARN, 
					  CCPayloadHandler ccPayloadHandler) throws Exception {
	super();
	
	try {
		this.mySourceIp = InetAddress.getLocalHost().getHostAddress();
	} catch(Exception e) {
		logger.error("Error getting local inet address: " + e.getMessage());
	}
	
	mySourceIdentifier = determineHostName() + "-" +uuid;
	this.snsControlTopicName = snsControlTopicName;
	
	if (callerIsMaster) {
		canDestroyTopic = true;
		this.snsControlTopicName += "-" + mySourceIdentifier;
	}
	
	this.ccPayloadHandler = ccPayloadHandler;
	
	sqsClient = new AmazonSQSClient(new BasicAWSCredentials(awsAccessKey, awsSecretKey));
	snsClient = new AmazonSNSClient(new BasicAWSCredentials(awsAccessKey, awsSecretKey));
	
	
	this.connectToTopic(callerIsMaster, 1000, userAccountPrincipalId, userARN);
	


}
 

/**
 * Create client using credentials provider. This is the preferred way for creating clients
 */
@Override
protected AmazonSQSClient createClient(final ProcessContext context, final AWSCredentialsProvider credentialsProvider, final ClientConfiguration config) {
    getLogger().info("Creating client using aws credentials provider ");

    return new AmazonSQSClient(credentialsProvider, config);
}
 

@Override
public void onTrigger(final ProcessContext context, final ProcessSession session) {
    FlowFile flowFile = session.get();
    if (flowFile == null) {
        return;
    }

    final String queueUrl = context.getProperty(QUEUE_URL).evaluateAttributeExpressions(flowFile).getValue();

    final AmazonSQSClient client = getClient();
    final DeleteMessageBatchRequest request = new DeleteMessageBatchRequest();
    request.setQueueUrl(queueUrl);

    final List<DeleteMessageBatchRequestEntry> entries = new ArrayList<>();
    final DeleteMessageBatchRequestEntry entry = new DeleteMessageBatchRequestEntry();
    String receiptHandle = context.getProperty(RECEIPT_HANDLE).evaluateAttributeExpressions(flowFile).getValue();
    entry.setReceiptHandle(receiptHandle);
    String entryId = flowFile.getAttribute(CoreAttributes.UUID.key());
    entry.setId(entryId);
    entries.add(entry);
    request.setEntries(entries);

    try {
        DeleteMessageBatchResult response = client.deleteMessageBatch(request);

        // check for errors
        if (!response.getFailed().isEmpty()) {
            throw new ProcessException(response.getFailed().get(0).toString());
        }

        getLogger().info("Successfully deleted message from SQS for {}", new Object[] { flowFile });
        session.transfer(flowFile, REL_SUCCESS);
    } catch (final Exception e) {
        getLogger().error("Failed to delete message from SQS due to {}", new Object[] { e });
        flowFile = session.penalize(flowFile);
        session.transfer(flowFile, REL_FAILURE);
        return;
    }
}
 

@Before
public void setUp() {
    mockSQSClient = Mockito.mock(AmazonSQSClient.class);
    mockGetSQS = new GetSQS() {
        protected AmazonSQSClient getClient() {
            actualSQSClient = client;
            return mockSQSClient;
        }
    };
    runner = TestRunners.newTestRunner(mockGetSQS);
}
 

@Before
public void setUp() {
    mockSQSClient = Mockito.mock(AmazonSQSClient.class);
    DeleteMessageBatchResult mockResponse = Mockito.mock(DeleteMessageBatchResult.class);
    Mockito.when(mockSQSClient.deleteMessageBatch(Mockito.any())).thenReturn(mockResponse);
    Mockito.when(mockResponse.getFailed()).thenReturn(new ArrayList<>());
    mockDeleteSQS = new DeleteSQS() {
        @Override
        protected AmazonSQSClient getClient() {
            return mockSQSClient;
        }
    };
    runner = TestRunners.newTestRunner(mockDeleteSQS);
}
 

@Before
public void setUp() {
    mockSQSClient = Mockito.mock(AmazonSQSClient.class);
    mockPutSQS = new PutSQS() {
        @Override
        protected AmazonSQSClient getClient() {
            actualSQSClient = client;
            return mockSQSClient;
        }
    };
    runner = TestRunners.newTestRunner(mockPutSQS);
}
 

/**
 * Create client using credentials provider. This is the preferred way for creating clients
 */
@Override
protected AmazonSQSClient createClient(final ProcessContext context, final AWSCredentialsProvider credentialsProvider, final ClientConfiguration config) {
    getLogger().info("Creating client using aws credentials provider ");

    return new AmazonSQSClient(credentialsProvider, config);
}
 

/**
 * Create client using AWSCredentials
 *
 * @deprecated use {@link #createClient(ProcessContext, AWSCredentialsProvider, ClientConfiguration)} instead
 */
@Override
protected AmazonSQSClient createClient(final ProcessContext context, final AWSCredentials credentials, final ClientConfiguration config) {
    getLogger().info("Creating client using aws credentials ");

    return new AmazonSQSClient(credentials, config);
}
 

@JsonCreator
public SQSNotice(
        @JsonProperty("queues") List<String> queues,
        @JsonProperty("region") @JacksonInject("region") String region,
        @JsonProperty("connectionTimeout") int connectionTimeout,
        @JsonProperty("maxConnections") int maxConnections,
        @JsonProperty("socketTimeout") int socketTimeout,
        @JsonProperty("maxRetries") int maxRetries,
        @JsonProperty("enableBase64Encoding") boolean enableBase64Encoding,
        @JacksonInject AmazonSQSClient sqsClient,
        @JacksonInject AWSCredentialsProvider credentialsProvider) {
    this.queues = queues;
    this.region = region;

    this.enableBase64Encoding = enableBase64Encoding;
    this.sqsClient = sqsClient;
    this.credentialsProvider = credentialsProvider;

    Preconditions.checkArgument(queues.size() > 0);
    Preconditions.checkNotNull(region);

    clientConfig = new ClientConfiguration();
    if (connectionTimeout > 0) {
        clientConfig = clientConfig.withConnectionTimeout(connectionTimeout);
    }
    if (maxConnections > 0) {
        clientConfig = clientConfig.withMaxConnections(maxConnections);
    }
    if (socketTimeout > 0) {
        clientConfig = clientConfig.withSocketTimeout(socketTimeout);
    }
    if (maxRetries > 0) {
        clientConfig = clientConfig.withMaxErrorRetry(maxRetries);
    }

    Monitors.registerObject(Joiner.on('_').join(queues), this);
}
 

@Override
public AmazonSQSClient get() {
    AmazonSQSClient client = mock(AmazonSQSClient.class);
    doReturn(new SendMessageResult()).when(client).sendMessage(any(SendMessageRequest.class));

    ReceiveMessageResult result = new ReceiveMessageResult();
    result.setMessages(Arrays.asList(new Message[]{new Message().withBody("receivedMessage")}));
    doReturn(result).when(client).receiveMessage(any(ReceiveMessageRequest.class));
    doReturn(new GetQueueUrlResult().withQueueUrl("queueURL")).when(client).getQueueUrl(any(GetQueueUrlRequest.class));

    return client;
}
 

public SqsTest invoke() throws IOException {
    ObjectMapper mapper = injector.getInstance(DefaultObjectMapper.class);
    AmazonSQSClient client = injector.getInstance(AmazonSQSClient.class);

    queueNotice = mapper.readValue(desc, new TypeReference<Notice>() {});
    queueNotice.init();
    queueNotice.send("message");
    captor = ArgumentCaptor.forClass(SendMessageRequest.class);
    verify(client).sendMessage(captor.capture());
    return this;
}
 

public static void assertNoStaleQueue(AmazonSQSClient client, String when) {
    List<String> staleInstances = client.listQueues().getQueueUrls().stream() //
            .map(url -> url.substring(url.lastIndexOf(':') + 1))
            .filter(name -> !name.startsWith(SQSIntegrationTest.class.getSimpleName())
                    || System.currentTimeMillis() - AWSUtils.toEpochMillis(name) > AWSUtils.HOUR) //
            .collect(Collectors.toList());
    Assert.assertEquals(String.format("Found stale SQS queues %s running the test: %s", when, staleInstances), 0,
            staleInstances.size());
}
 

public static AmazonSQSClient createSQSClient() {
    BasicCredentialsProvider credentials = BasicCredentialsProvider.standard();
    AmazonSQSClient client = !credentials.isValid() ? null : (AmazonSQSClient)
            AmazonSQSClientBuilder.standard()
            .withCredentials(credentials)
            .withRegion("eu-west-1").build();
    return client;
}
 

/**
 * Create the SQS client for the specified settings
 */
private AmazonSQSClient createSQSClient( final Region region ) {

    final UsergridAwsCredentialsProvider ugProvider = new UsergridAwsCredentialsProvider();
    final AmazonSQSClient sqs =
        new AmazonSQSClient( ugProvider.getCredentials(), clientConfiguration );

    sqs.setRegion( region );

    return sqs;
}
 

/********************** Member Functions **************************/

	public AwsGlacier(String region) {
		// Get credentials from credentials file, environment variable, or 
		// Java property. 
		// See http://docs.aws.amazon.com/AWSJavaSDK/latest/javadoc/com/amazonaws/auth/DefaultAWSCredentialsProviderChain.html
		AWSCredentialsProviderChain credentialsProvider = 
				new DefaultAWSCredentialsProviderChain();
		credentials = credentialsProvider.getCredentials();
		logger.debug("Read in credentials AWSAccessKeyId={} AWSSecretKey={}...",
				credentials.getAWSAccessKeyId(), 
				credentials.getAWSSecretKey().substring(0, 4));
		
		// Create the glacier client and set to specified region.
		glacierClient = new AmazonGlacierClient(credentials);
		glacierClient.setEndpoint("https://glacier." + region + ".amazonaws.com");
		
		// Set up params needed for retrieving vault inventory
        sqsClient = new AmazonSQSClient(credentials);
        sqsClient.setEndpoint("https://sqs." + region + ".amazonaws.com");
        snsClient = new AmazonSNSClient(credentials);
        snsClient.setEndpoint("https://sns." + region + ".amazonaws.com");

        // Create the ArchiveTransferManager used for uploading and 
        // downloading files. Need to use ArchiveTransferManager constructor 
        // that allows one to specify sqsClient & snsClient so that they have
        // the proper region. If use ArchiveTransferManager without specifying
        // sqs and sns clients then default ones are constructed, but these
        // use the default Virginia region, which is wrong.
		atm = new ArchiveTransferManager(glacierClient, sqsClient, snsClient);
}
 

/**
 * Create client using AWSCredentials
 *
 * @deprecated use {@link #createClient(ProcessContext, AWSCredentialsProvider, ClientConfiguration)} instead
 */
@Override
protected AmazonSQSClient createClient(final ProcessContext context, final AWSCredentials credentials, final ClientConfiguration config) {
    getLogger().info("Creating client using aws credentials ");

    return new AmazonSQSClient(credentials, config);
}
 

@ProvidesIntoMap
@StringMapKey("sqs")
@Singleton
@Named(EVENT_QUEUE_PROVIDERS_QUALIFIER)
public EventQueueProvider getSQSEventQueueProvider(AmazonSQSClient amazonSQSClient, Configuration config) {
	return new SQSEventQueueProvider(amazonSQSClient, config);
}
 

@Provides
public Map<Status, ObservableQueue> getQueues(Configuration config, AWSCredentialsProvider acp) {
	
	String stack = "";
	if(config.getStack() != null && config.getStack().length() > 0) {
		stack = config.getStack() + "_";
	}
	Status[] statuses = new Status[]{Status.COMPLETED, Status.FAILED};
	Map<Status, ObservableQueue> queues = new HashMap<>();
	for(Status status : statuses) {
		String queueName = config.getProperty("workflow.listener.queue.prefix", config.getAppId() + "_sqs_notify_" + stack + status.name());
		AmazonSQSClient client = new AmazonSQSClient(acp);
		Builder builder = new SQSObservableQueue.Builder().withClient(client).withQueueName(queueName);
		
		String auth = config.getProperty("workflow.listener.queue.authorizedAccounts", "");
		String[] accounts = auth.split(",");
		for(String accountToAuthorize : accounts) {
			accountToAuthorize = accountToAuthorize.trim();
			if(accountToAuthorize.length() > 0) {
				builder.addAccountToAuthorize(accountToAuthorize.trim());
			}
		}
		ObservableQueue queue = builder.build();
		queues.put(status, queue);
	}
	
	return queues;
}
 

private SQSObservableQueue(String queueName, AmazonSQSClient client, int visibilityTimeoutInSeconds, int batchSize, int pollTimeInMS, List<String> accountsToAuthorize) {
	this.queueName = queueName;
	this.client = client;
	this.visibilityTimeoutInSeconds = visibilityTimeoutInSeconds;
	this.batchSize = batchSize;
	this.pollTimeInMS = pollTimeInMS;
	this.queueURL = getOrCreateQueue();
	addPolicy(accountsToAuthorize);
}
 

@Inject
public SQSEventQueueProvider(AmazonSQSClient client, Configuration config) {
	this.client = client;
	this.batchSize = config.getIntProperty("workflow.event.queues.sqs.batchSize", 1);
	this.pollTimeInMS = config.getIntProperty("workflow.event.queues.sqs.pollTimeInMS", 100);
	this.visibilityTimeoutInSeconds = config.getIntProperty("workflow.event.queues.sqs.visibilityTimeoutInSeconds", 60);
}
 

@Test
public void testException() {
    com.amazonaws.services.sqs.model.Message message = new com.amazonaws.services.sqs.model.Message().withMessageId("test")
            .withBody("")
            .withReceiptHandle("receiptHandle");
    Answer<?> answer = (Answer<ReceiveMessageResult>) invocation -> new ReceiveMessageResult();

    AmazonSQSClient client = mock(AmazonSQSClient.class);
    when(client.listQueues(any(ListQueuesRequest.class))).thenReturn(new ListQueuesResult().withQueueUrls("junit_queue_url"));
    when(client.receiveMessage(any(ReceiveMessageRequest.class))).thenThrow(new RuntimeException("Error in SQS communication"))
            .thenReturn(new ReceiveMessageResult().withMessages(message))
            .thenAnswer(answer);

    SQSObservableQueue queue = new SQSObservableQueue.Builder()
            .withQueueName("junit")
            .withClient(client).build();

    List<Message> found = new LinkedList<>();
    Observable<Message> observable = queue.observe();
    assertNotNull(observable);
    observable.subscribe(found::add);

    Uninterruptibles.sleepUninterruptibly(1000, TimeUnit.MILLISECONDS);

    assertEquals(1, found.size());

}
 

@Provides
@Singleton
protected AmazonSQS provideAmazonSQS(Region region, AWSCredentialsProvider credentialsProvider) {
    AmazonSQS amazonSQS = new AmazonSQSClient(credentialsProvider);
    amazonSQS.setRegion(region);
    return amazonSQS;
}
 

private void createSQSClient()
{
  createElasticMQMockServer();
  // just use x and y as access and secret keys respectively: not checked by the mock server
  sqs = new AmazonSQSClient(new BasicAWSCredentials("x", "y"));
  // have it talk to the embedded server
  sqs.setEndpoint("http://localhost:9324");
}