com.microsoft.azure.storage.queue.CloudQueueClient Java Examples

/**
 * This method create a queue if it doesn't exist
 */
public boolean createQueueIfNotExists(String queueName) throws InvalidKeyException, URISyntaxException, StorageException {
    CloudQueueClient client = connection.getCloudStorageAccount().createCloudQueueClient();
    CloudQueue queueRef = client.getQueueReference(queueName);
    boolean creationResult;
    try {
        creationResult = queueRef.createIfNotExists(null, AzureStorageUtils.getTalendOperationContext());
    } catch (StorageException e) {
        if (!e.getErrorCode().equals(StorageErrorCodeStrings.QUEUE_BEING_DELETED)) {
            throw e;
        }
        LOGGER.warn(messages.getMessage("error.QueueDeleted", queueRef.getName()));
        // Documentation doesn't specify how many seconds at least to wait.
        // 40 seconds before retrying.
        // See https://docs.microsoft.com/en-us/rest/api/storageservices/fileservices/delete-queue3
        try {
            Thread.sleep(40000);
        } catch (InterruptedException eint) {
            throw new RuntimeException(messages.getMessage("error.InterruptedException"));
        }
        creationResult = queueRef.createIfNotExists(null, AzureStorageUtils.getTalendOperationContext());
        LOGGER.debug(messages.getMessage("debug.QueueCreated", queueRef.getName()));
    }

    return creationResult;
}
 

/**
 * Creates a new Queue service client.
 * 
 * @return A client object that uses the Queue service endpoint.
 */
public CloudQueueClient createCloudQueueClient() {
    if (this.getQueueStorageUri() == null) {
        throw new IllegalArgumentException(SR.QUEUE_ENDPOINT_NOT_CONFIGURED);
    }

    if (this.credentials == null) {
        throw new IllegalArgumentException(SR.MISSING_CREDENTIALS);
    }

    if (!StorageCredentialsHelper.canCredentialsGenerateClient(this.credentials)) {
        
        throw new IllegalArgumentException(SR.CREDENTIALS_CANNOT_SIGN_REQUEST);
    }
    
    return new CloudQueueClient(this.getQueueStorageUri(), this.getCredentials());
}
 

@Test
public void testCloudStorageAccountClientUriVerify() throws URISyntaxException, StorageException {
    StorageCredentialsAccountAndKey cred = new StorageCredentialsAccountAndKey(ACCOUNT_NAME, ACCOUNT_KEY);
    CloudStorageAccount cloudStorageAccount = new CloudStorageAccount(cred, true);

    CloudBlobClient blobClient = cloudStorageAccount.createCloudBlobClient();
    CloudBlobContainer container = blobClient.getContainerReference("container1");
    assertEquals(cloudStorageAccount.getBlobEndpoint().toString() + "/container1", container.getUri().toString());

    CloudQueueClient queueClient = cloudStorageAccount.createCloudQueueClient();
    CloudQueue queue = queueClient.getQueueReference("queue1");
    assertEquals(cloudStorageAccount.getQueueEndpoint().toString() + "/queue1", queue.getUri().toString());

    CloudTableClient tableClient = cloudStorageAccount.createCloudTableClient();
    CloudTable table = tableClient.getTableReference("table1");
    assertEquals(cloudStorageAccount.getTableEndpoint().toString() + "/table1", table.getUri().toString());

    CloudFileClient fileClient = cloudStorageAccount.createCloudFileClient();
    CloudFileShare share = fileClient.getShareReference("share1");
    assertEquals(cloudStorageAccount.getFileEndpoint().toString() + "/share1", share.getUri().toString());
}
 

private static void testQueueDownloadAttributes(LocationMode optionsLocationMode, LocationMode clientLocationMode,
        StorageLocation initialLocation, List<RetryContext> retryContextList, List<RetryInfo> retryInfoList)
        throws URISyntaxException, StorageException {
    CloudQueueClient client = TestHelper.createCloudQueueClient();
    CloudQueue queue = client.getQueueReference(QueueTestHelper.generateRandomQueueName());

    MultiLocationTestHelper helper = new MultiLocationTestHelper(queue.getServiceClient().getStorageUri(),
            initialLocation, retryContextList, retryInfoList);

    queue.getServiceClient().getDefaultRequestOptions().setLocationMode(clientLocationMode);
    QueueRequestOptions options = new QueueRequestOptions();

    options.setLocationMode(optionsLocationMode);
    options.setRetryPolicyFactory(helper.retryPolicy);

    try {
        queue.downloadAttributes(options, helper.operationContext);
    }
    catch (StorageException ex) {
        assertEquals(HttpURLConnection.HTTP_NOT_FOUND, ex.getHttpStatusCode());
    }
    finally {
        helper.close();
    }
}
 

private ServiceProperties callDownloadServiceProperties(ServiceClient client) throws StorageException {
    if (client.getClass().equals(CloudBlobClient.class)) {
        CloudBlobClient blobClient = (CloudBlobClient) client;
        return blobClient.downloadServiceProperties();
    }
    else if (client.getClass().equals(CloudTableClient.class)) {
        CloudTableClient tableClient = (CloudTableClient) client;
        return tableClient.downloadServiceProperties();
    }
    else if (client.getClass().equals(CloudQueueClient.class)) {
        CloudQueueClient queueClient = (CloudQueueClient) client;
        return queueClient.downloadServiceProperties();
    }
    else {
        fail();
    }
    return null;
}
 

private void callUploadServiceProps(
        ServiceClient client, ServiceProperties props, FileServiceProperties fileProps)
        throws StorageException, InterruptedException {

    if (client.getClass().equals(CloudBlobClient.class)) {
        ((CloudBlobClient) client).uploadServiceProperties(props);
    }
    else if (client.getClass().equals(CloudTableClient.class)) {
        ((CloudTableClient) client).uploadServiceProperties(props);
    }
    else if (client.getClass().equals(CloudQueueClient.class)) {
        ((CloudQueueClient) client).uploadServiceProperties(props);
    }
    else if (client.getClass().equals(CloudFileClient.class)) {
        ((CloudFileClient) client).uploadServiceProperties(fileProps);
    }
    else {
        fail();
    }

    Thread.sleep(30000);
}
 

@Test
@Category({ DevFabricTests.class, DevStoreTests.class, SecondaryTests.class })
public void testQueueMaximumExecutionTime() throws URISyntaxException, StorageException {
    OperationContext opContext = new OperationContext();
    setDelay(opContext, 2500);

    // set the maximum execution time
    QueueRequestOptions options = new QueueRequestOptions();
    options.setMaximumExecutionTimeInMs(2000);

    // set the location mode to secondary, secondary request should fail
    // so set the timeout low to save time failing (or fail with a timeout)
    options.setLocationMode(LocationMode.SECONDARY_THEN_PRIMARY);
    options.setTimeoutIntervalInMs(1000);

    CloudQueueClient queueClient = TestHelper.createCloudQueueClient();
    CloudQueue queue = queueClient.getQueueReference(generateRandomName("queue"));

    try {
        // 1. download attributes will fail as the queue does not exist
        // 2. the executor will attempt to retry as it is accessing secondary
        // 3. maximum execution time should prevent the retry from being made
        queue.downloadAttributes(options, opContext);
        fail("Maximum execution time was reached but request did not fail.");
    }
    catch (StorageException e) {
        assertEquals(SR.MAXIMUM_EXECUTION_TIMEOUT_EXCEPTION, e.getMessage());
    }
}
 

@Before
public void setUpAzureQueueStorageIT() throws Exception {
    String queueName = String.format("%s-%s", TEST_QUEUE_NAME_PREFIX, UUID.randomUUID());
    CloudQueueClient cloudQueueClient = getStorageAccount().createCloudQueueClient();
    cloudQueue = cloudQueueClient.getQueueReference(queueName);
    cloudQueue.createIfNotExists();

    runner.setProperty(AbstractAzureQueueStorage.QUEUE, queueName);
}
 

@Override
public void onTrigger(ProcessContext context, ProcessSession session) throws ProcessException {
    FlowFile flowFile = session.get();
    if (flowFile == null) {
        return;
    }

    final long startNanos = System.nanoTime();

    final ByteArrayOutputStream baos = new ByteArrayOutputStream();
    session.exportTo(flowFile, baos);
    final String flowFileContent = baos.toString();

    CloudQueueMessage message = new CloudQueueMessage(flowFileContent);
    CloudQueueClient cloudQueueClient;
    CloudQueue cloudQueue;

    final int ttl = context.getProperty(TTL).asTimePeriod(TimeUnit.SECONDS).intValue();
    final int delay = context.getProperty(VISIBILITY_DELAY).asTimePeriod(TimeUnit.SECONDS).intValue();
    final String queue = context.getProperty(QUEUE).evaluateAttributeExpressions(flowFile).getValue().toLowerCase();

    try {
        cloudQueueClient = createCloudQueueClient(context, flowFile);
        cloudQueue = cloudQueueClient.getQueueReference(queue);

        final OperationContext operationContext = new OperationContext();
        AzureStorageUtils.setProxy(operationContext, context);

        cloudQueue.addMessage(message, ttl, delay, null, operationContext);
    } catch (URISyntaxException | StorageException e) {
        getLogger().error("Failed to write the message to Azure Queue Storage due to {}", new Object[]{e});
        flowFile = session.penalize(flowFile);
        session.transfer(flowFile, REL_FAILURE);
        return;
    }

    session.transfer(flowFile, REL_SUCCESS);
    final long transmissionMillis = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startNanos);
    session.getProvenanceReporter().send(flowFile, cloudQueue.getUri().toString(), transmissionMillis);
}
 

protected final CloudQueueClient createCloudQueueClient(final ProcessContext context, final FlowFile flowFile) throws URISyntaxException {
    final AzureStorageCredentialsDetails storageCredentialsDetails = AzureStorageUtils.getStorageCredentialsDetails(context, flowFile);
    final CloudStorageAccount cloudStorageAccount =
        new CloudStorageAccount(
            storageCredentialsDetails.getStorageCredentials(),
            true,
            storageCredentialsDetails.getStorageSuffix(),
            storageCredentialsDetails.getStorageAccountName()
        );
    final CloudQueueClient cloudQueueClient = cloudStorageAccount.createCloudQueueClient();

    return cloudQueueClient;
}
 

@Test
public void testCloudStorageAccountClientMethods() throws URISyntaxException {
    StorageCredentialsAccountAndKey cred = new StorageCredentialsAccountAndKey(ACCOUNT_NAME, ACCOUNT_KEY);

    CloudStorageAccount account = new CloudStorageAccount(cred, false);
    CloudBlobClient blob = account.createCloudBlobClient();
    CloudQueueClient queue = account.createCloudQueueClient();
    CloudTableClient table = account.createCloudTableClient();
    CloudFileClient file = account.createCloudFileClient();

    // check endpoints
    assertEquals("Blob endpoint doesn't match account", account.getBlobEndpoint(), blob.getEndpoint());
    assertEquals("Queue endpoint doesn't match account", account.getQueueEndpoint(), queue.getEndpoint());
    assertEquals("Table endpoint doesn't match account", account.getTableEndpoint(), table.getEndpoint());
    assertEquals("File endpoint doesn't match account", account.getFileEndpoint(), file.getEndpoint());

    // check storage uris
    assertEquals("Blob endpoint doesn't match account", account.getBlobStorageUri(), blob.getStorageUri());
    assertEquals("Queue endpoint doesn't match account", account.getQueueStorageUri(), queue.getStorageUri());
    assertEquals("Table endpoint doesn't match account", account.getTableStorageUri(), table.getStorageUri());
    assertEquals("File endpoint doesn't match account", account.getFileStorageUri(), file.getStorageUri());

    // check creds
    assertEquals("Blob creds don't match account", account.getCredentials(), blob.getCredentials());
    assertEquals("Queue creds don't match account", account.getCredentials(), queue.getCredentials());
    assertEquals("Table creds don't match account", account.getCredentials(), table.getCredentials());
    assertEquals("File creds don't match account", account.getCredentials(), file.getCredentials());
}
 

public static CloudQueueClient createCloudQueueClient(SharedAccessAccountPolicy policy, boolean useHttps)
        throws StorageException, InvalidKeyException, URISyntaxException {

    CloudStorageAccount sasAccount = getAccount();
    final String token = sasAccount.generateSharedAccessSignature(policy);
    final StorageCredentials creds =
            new StorageCredentialsSharedAccessSignature(token);
    
    final URI queueUri = new URI(useHttps ? "https" : "http", sasAccount.getQueueEndpoint().getAuthority(), 
    		sasAccount.getQueueEndpoint().getPath(), sasAccount.getQueueEndpoint().getQuery(), null);
    
    sasAccount = new CloudStorageAccount(creds, sasAccount.getBlobEndpoint(), queueUri,
            sasAccount.getTableEndpoint(), sasAccount.getFileEndpoint());
    return sasAccount.createCloudQueueClient();
}
 

@Test
public void testUserAgentString() throws URISyntaxException, StorageException {
    // Test with a blob request
    CloudBlobClient blobClient = TestHelper.createCloudBlobClient();
    CloudBlobContainer container = blobClient.getContainerReference("container1");
    OperationContext sendingRequestEventContext = new OperationContext();
    sendingRequestEventContext.getSendingRequestEventHandler().addListener(new StorageEvent<SendingRequestEvent>() {

        @Override
        public void eventOccurred(SendingRequestEvent eventArg) {
            assertEquals(
                    Constants.HeaderConstants.USER_AGENT_PREFIX
                            + "/"
                            + Constants.HeaderConstants.USER_AGENT_VERSION
                            + " "
                            + String.format(Utility.LOCALE_US, "(Android %s; %s; %s)",
                            android.os.Build.VERSION.RELEASE, android.os.Build.BRAND,
                            android.os.Build.MODEL), ((HttpURLConnection) eventArg.getConnectionObject())
                            .getRequestProperty(Constants.HeaderConstants.USER_AGENT));
        }
    });
    container.exists(null, null, sendingRequestEventContext);

    // Test with a queue request
    CloudQueueClient queueClient = TestHelper.createCloudQueueClient();
    CloudQueue queue = queueClient.getQueueReference("queue1");
    queue.exists(null, sendingRequestEventContext);

    // Test with a table request
    CloudTableClient tableClient = TestHelper.createCloudTableClient();
    CloudTable table = tableClient.getTableReference("table1");
    table.exists(null, sendingRequestEventContext);
}
 

public void deleteMessage(String queueName, CloudQueueMessage message)
        throws InvalidKeyException, URISyntaxException, StorageException {

    CloudQueueClient client = connection.getCloudStorageAccount().createCloudQueueClient();
    CloudQueue queueRef = client.getQueueReference(queueName);
    queueRef.deleteMessage(message, null, AzureStorageUtils.getTalendOperationContext());
}
 

public Iterable<CloudQueueMessage> retrieveMessages(String queueName, int numberOfMessages, int visibilityTimeoutInSeconds)
        throws InvalidKeyException, URISyntaxException, StorageException {

    CloudQueueClient client = connection.getCloudStorageAccount().createCloudQueueClient();
    CloudQueue queueRef = client.getQueueReference(queueName);
    return queueRef.retrieveMessages(numberOfMessages, visibilityTimeoutInSeconds, null, AzureStorageUtils.getTalendOperationContext());
}
 

public Iterable<CloudQueueMessage> peekMessages(String queueName, int numberOfMessages)
        throws InvalidKeyException, URISyntaxException, StorageException {

    CloudQueueClient client = connection.getCloudStorageAccount().createCloudQueueClient();
    CloudQueue queueRef = client.getQueueReference(queueName);
    return queueRef.peekMessages(numberOfMessages, null, AzureStorageUtils.getTalendOperationContext());
}
 

@Override
public List<NamedThing> getSchemaNames(RuntimeContainer container) throws IOException {
    List<NamedThing> result = new ArrayList<>();
    try {
        CloudQueueClient client = getStorageQueueClient(container);
        for (CloudQueue q : client.listQueues()) {
            result.add(new SimpleNamedThing(q.getName(), q.getName()));
        }
    } catch (InvalidKeyException | URISyntaxException e) {
        throw new ComponentException(e);
    }
    return result;
}
 

public static CloudQueueClient createCloudQueueClient() throws StorageException {
	return getAccount().createCloudQueueClient();
}
 

public CloudQueueClient getStorageQueueClient(RuntimeContainer runtime) throws InvalidKeyException, URISyntaxException {
    return getStorageAccount(runtime).createCloudQueueClient();
}
 

public void clear(String queueName) throws InvalidKeyException, URISyntaxException, StorageException {
    CloudQueueClient client = connection.getCloudStorageAccount().createCloudQueueClient();
    CloudQueue queueRef = client.getQueueReference(queueName);
    queueRef.clear(null, AzureStorageUtils.getTalendOperationContext());
}
 

public long getApproximateMessageCount(String queueName) throws InvalidKeyException, URISyntaxException, StorageException {
    CloudQueueClient client = connection.getCloudStorageAccount().createCloudQueueClient();
    CloudQueue queueRef = client.getQueueReference(queueName);
    return queueRef.getApproximateMessageCount();
}
 

public Iterable<CloudQueue> listQueues() throws InvalidKeyException, URISyntaxException {

        CloudQueueClient client = connection.getCloudStorageAccount().createCloudQueueClient();
        return client.listQueues(null, QueueListingDetails.NONE, null, AzureStorageUtils.getTalendOperationContext());
    }
 

public boolean deleteQueueIfExists(String queueName) throws InvalidKeyException, URISyntaxException, StorageException {
    CloudQueueClient client = connection.getCloudStorageAccount().createCloudQueueClient();
    CloudQueue queueRef = client.getQueueReference(queueName);
    return queueRef.deleteIfExists(null, AzureStorageUtils.getTalendOperationContext());
}