com.microsoft.azure.storage.CloudStorageAccount Java Examples

/**
 * Upload image file to Azure storage with specified name.
 *
 * @param file     image file object
 * @param fileName specified file name
 * @return relative path of the created image blob
 */
public String uploadToAzureStorage(ApplicationContext applicationContext, MultipartFile file, String fileName) {
    String uri = null;

    try {
        CloudStorageAccount storageAccount =
                (CloudStorageAccount) applicationContext.getBean("cloudStorageAccount");
        CloudBlobClient blobClient = storageAccount.createCloudBlobClient();

        setupContainer(blobClient, this.thumbnailImageContainer);
        CloudBlobContainer originalImageContainer = setupContainer(blobClient, this.originalImageContainer);

        if (originalImageContainer != null) {
            CloudBlockBlob blob = originalImageContainer.getBlockBlobReference(fileName);
            blob.upload(file.getInputStream(), file.getSize());

            uri = blob.getUri().getPath();
        }
    } catch (Exception e) {
        e.printStackTrace();
        logger.error("Error uploading image: " + e.getMessage());
    }

    return uri;
}
 

private CloudBlobContainer getCloudBlobContainer(String accountName, String accountKey, String containerName) {
  CloudBlobContainer container = null;

  if (StringUtils.isNotBlank(containerName)) {
    final String storageConnectionString = "DefaultEndpointsProtocol=https"
      + ";AccountName=" + accountName
      + ";AccountKey=" + accountKey;

    try {
      final CloudStorageAccount account = CloudStorageAccount.parse(storageConnectionString);
      CloudBlobClient serviceClient = account.createCloudBlobClient();

      container = serviceClient.getContainerReference(containerName);
      container.createIfNotExists();
    } catch (StorageException | URISyntaxException | InvalidKeyException e) {
      logger.error("Error connecting to container for account {} and container name {}", accountName, containerName, e);
    }
  }

  return container;
}
 

public void init(ZeppelinConfiguration conf) throws IOException {
  this.conf = conf;
  user = conf.getString(ZeppelinConfiguration.ConfVars.ZEPPELIN_NOTEBOOK_AZURE_USER);
  shareName = conf.getString(ZeppelinConfiguration.ConfVars.ZEPPELIN_NOTEBOOK_AZURE_SHARE);

  try {
    CloudStorageAccount account = CloudStorageAccount.parse(
        conf.getString(ZeppelinConfiguration.ConfVars.ZEPPELIN_NOTEBOOK_AZURE_CONNECTION_STRING));
    CloudFileClient client = account.createCloudFileClient();
    CloudFileShare share = client.getShareReference(shareName);
    share.createIfNotExists();

    CloudFileDirectory userDir = StringUtils.isBlank(user) ?
        share.getRootDirectoryReference() :
        share.getRootDirectoryReference().getDirectoryReference(user);
    userDir.createIfNotExists();

    rootDir = userDir.getDirectoryReference("notebook");
    rootDir.createIfNotExists();
  } catch (Exception e) {
    throw new IOException(e);
  }
}
 

private static CloudBlobContainer setUpStorageAccount(String connectionString, String containerName) {
    try {
        CloudStorageAccount account = CloudStorageAccount.parse(connectionString);
        // Create a blob service client
        CloudBlobClient blobClient = account.createCloudBlobClient();
        CloudBlobContainer container = blobClient.getContainerReference(containerName);
        container.createIfNotExists();
        BlobContainerPermissions containerPermissions = new BlobContainerPermissions();
        // Include public access in the permissions object
        containerPermissions.setPublicAccess(BlobContainerPublicAccessType.CONTAINER);
        // Set the permissions on the container
        container.uploadPermissions(containerPermissions);
        return container;
    } catch (StorageException | URISyntaxException | InvalidKeyException e) {
        throw new RuntimeException(e);
    }

}
 

private static CloudBlobContainer setUpStorageAccount(String connectionString, String containerName) {
    try {
        CloudStorageAccount account = CloudStorageAccount.parse(connectionString);
        // Create a blob service client
        CloudBlobClient blobClient = account.createCloudBlobClient();
        CloudBlobContainer container = blobClient.getContainerReference(containerName);
        container.createIfNotExists();
        BlobContainerPermissions containerPermissions = new BlobContainerPermissions();
        // Include public access in the permissions object
        containerPermissions.setPublicAccess(BlobContainerPublicAccessType.CONTAINER);
        // Set the permissions on the container
        container.uploadPermissions(containerPermissions);
        return container;
    } catch (StorageException | URISyntaxException | InvalidKeyException e) {
        throw new RuntimeException(e);
    }

}
 

public TableClient create() {
    try {
        final CloudStorageAccount storageAccount =
                new CloudStorageAccount(azureTableConfiguration.getStorageCredentialsAccountAndKey(), true);
        final CloudTableClient cloudTableClient = storageAccount.createCloudTableClient();
        final TableRequestOptions defaultOptions = new TableRequestOptions();
        defaultOptions.setRetryPolicyFactory(new RetryLinearRetry(
                Ints.checkedCast(azureTableConfiguration.getRetryInterval().toMilliseconds()),
                azureTableConfiguration.getRetryAttempts()));
        defaultOptions.setTimeoutIntervalInMs(Ints.checkedCast(azureTableConfiguration.getTimeout().toMilliseconds()));
        cloudTableClient.setDefaultRequestOptions(defaultOptions);
        return new TableClient(cloudTableClient);
    } catch (URISyntaxException err) {
        LOGGER.error("Failed to create a TableClient", err);
        throw new IllegalArgumentException(err);
    }
}
 

private Observable<CloudStorageAccount> getCloudStorageAsync(final StorageAccount storageAccount) {
    return storageAccount.getKeysAsync()
            .flatMapIterable(new Func1<List<StorageAccountKey>, Iterable<StorageAccountKey>>() {
                @Override
                public Iterable<StorageAccountKey> call(List<StorageAccountKey> storageAccountKeys) {
                    return storageAccountKeys;
                }
            })
            .last()
            .map(new Func1<StorageAccountKey, CloudStorageAccount>() {
                @Override
                public CloudStorageAccount call(StorageAccountKey storageAccountKey) {
                    try {
                    return CloudStorageAccount.parse(Utils.getStorageConnectionString(storageAccount.name(), storageAccountKey.value(), manager().inner().restClient()));
                    } catch (URISyntaxException syntaxException) {
                        throw Exceptions.propagate(syntaxException);
                    } catch (InvalidKeyException keyException) {
                        throw Exceptions.propagate(keyException);
                    }
                }
            });
}
 

private Observable<Boolean> createContainerIfNotExistsAsync(final StorageAccount storageAccount,
                                                            final String containerName) {
    return getCloudStorageAsync(storageAccount)
            .flatMap(new Func1<CloudStorageAccount, Observable<Boolean>>() {
                @Override
                public Observable<Boolean> call(final CloudStorageAccount cloudStorageAccount) {
                    return Observable.fromCallable(new Callable<Boolean>() {
                        @Override
                        public Boolean call() {
                            CloudBlobClient blobClient = cloudStorageAccount.createCloudBlobClient();
                            try {
                                return blobClient.getContainerReference(containerName).createIfNotExists();
                            } catch (StorageException stgException) {
                                throw Exceptions.propagate(stgException);
                            } catch (URISyntaxException syntaxException) {
                                throw Exceptions.propagate(syntaxException);
                            }
                        }
                    }).subscribeOn(SdkContext.getRxScheduler());
                }
            });
}
 

@Before
public void addMockRules() {
  CloudStorageAccount cloudStorageAccount = mock(CloudStorageAccount.class);
  CloudBlobClient cloudBlobClient = mock(CloudBlobClient.class);
  CloudBlobContainer cloudBlobContainer = mock(CloudBlobContainer.class);

  ListBlobItem listBlobItem = mock(ListBlobItem.class);
  List<ListBlobItem> lst = new ArrayList<>();
  lst.add(listBlobItem);
  PowerMockito.mockStatic(CloudStorageAccount.class);
  try {
    doReturn(cloudStorageAccount).when(CloudStorageAccount.class, "parse", Mockito.anyString());
    doReturn(cloudBlobClient).when(cloudStorageAccount).createCloudBlobClient();
    doReturn(cloudBlobContainer).when(cloudBlobClient).getContainerReference(Mockito.anyString());
    doReturn(true).when(cloudBlobContainer).exists();
    when(cloudBlobContainer.listBlobs()).thenReturn(lst);
    when(listBlobItem.getUri()).thenReturn(new URI("http://www.google.com"));

  } catch (Exception e) {
    Assert.fail("Could not initalize mocks, underlying reason " + e.getLocalizedMessage());
  }
}
 

private void validateAdlsGen2FileSystem(SpiFileSystem fileSystem) throws URISyntaxException, InvalidKeyException, StorageException {
    CloudAdlsGen2View cloudFileSystem = (CloudAdlsGen2View) fileSystem.getCloudFileSystems().get(0);
    String accountName = cloudFileSystem.getAccountName();
    String accountKey = cloudFileSystem.getAccountKey();
    String connectionString = "DefaultEndpointsProtocol=https;AccountName="
            + accountName + ";AccountKey=" + accountKey + ";EndpointSuffix=core.windows.net";
    CloudStorageAccount storageAccount = CloudStorageAccount.parse(connectionString);
    CloudBlobClient blobClient = storageAccount.createCloudBlobClient();
    CloudBlobContainer containerReference = blobClient.getContainerReference(TEST_CONTAINER + System.nanoTime());
    try {
        containerReference.createIfNotExists();
        containerReference.delete();
    } catch (StorageException e) {
        if (e.getCause() instanceof UnknownHostException) {
            throw new CloudConnectorException("The provided account does not belong to a valid storage account");
        }
    }
}
 

private void validateWasbFileSystem(SpiFileSystem fileSystem) throws URISyntaxException, InvalidKeyException, StorageException {
    CloudWasbView cloudFileSystem = (CloudWasbView) fileSystem.getCloudFileSystems().get(0);
    String accountName = cloudFileSystem.getAccountName();
    String accountKey = cloudFileSystem.getAccountKey();
    String connectionString = "DefaultEndpointsProtocol=https;AccountName=" + accountName + ";AccountKey=" + accountKey;
    CloudStorageAccount storageAccount = CloudStorageAccount.parse(connectionString);
    CloudBlobClient blobClient = storageAccount.createCloudBlobClient();
    CloudBlobContainer containerReference = blobClient.getContainerReference(TEST_CONTAINER + System.nanoTime());
    try {
        containerReference.createIfNotExists();
        containerReference.delete();
    } catch (StorageException e) {
        if (e.getCause() instanceof UnknownHostException) {
            throw new CloudConnectorException("The provided account does not belong to a valid storage account");
        }
    }
}
 

public void init(ZeppelinConfiguration conf) throws IOException {
  this.conf = conf;
  user = conf.getString(ZeppelinConfiguration.ConfVars.ZEPPELIN_NOTEBOOK_AZURE_USER);
  shareName = conf.getString(ZeppelinConfiguration.ConfVars.ZEPPELIN_NOTEBOOK_AZURE_SHARE);

  try {
    CloudStorageAccount account = CloudStorageAccount.parse(
        conf.getString(ZeppelinConfiguration.ConfVars.ZEPPELIN_NOTEBOOK_AZURE_CONNECTION_STRING));
    CloudFileClient client = account.createCloudFileClient();
    CloudFileShare share = client.getShareReference(shareName);
    share.createIfNotExists();

    CloudFileDirectory userDir = StringUtils.isBlank(user) ?
        share.getRootDirectoryReference() :
        share.getRootDirectoryReference().getDirectoryReference(user);
    userDir.createIfNotExists();

    rootDir = userDir.getDirectoryReference("notebook");
    rootDir.createIfNotExists();
  } catch (Exception e) {
    throw new IOException(e);
  }
}
 

@Override
public void validate(ConfigProblemSetBuilder ps, AzsPersistentStore n) {
  String connectionString =
      "DefaultEndpointsProtocol=https;AccountName="
          + n.getStorageAccountName()
          + ";AccountKey="
          + secretSessionManager.decrypt(n.getStorageAccountKey());

  try {
    CloudStorageAccount storageAccount = CloudStorageAccount.parse(connectionString);

    CloudBlobContainer container =
        storageAccount.createCloudBlobClient().getContainerReference(n.getStorageContainerName());
    container.exists();
  } catch (Exception e) {
    ps.addProblem(
        Problem.Severity.ERROR,
        "Failed to connect to the Azure storage account \""
            + n.getStorageAccountName()
            + "\": "
            + e.getMessage());
    return;
  }
}
 

@Before
public void testCreateContainer() {
    try {
        // Retrieve storage account from connection-string.
        CloudStorageAccount storageAccount = CloudStorageAccount.parse(storageConnectionString);

        // Create the blob client.
        CloudBlobClient blobClient = storageAccount.createCloudBlobClient();

        // Get a reference to a container.
        // The container name must be lower case
        container = blobClient.getContainerReference(containerName);

        // Create the container if it does not exist.
        container.createIfNotExists();
    } catch (Exception e) {
        // Output the stack trace.
        e.printStackTrace();
    }
}
 

public VideoStorageAzure(List<StorageCredentials> accounts, String container, long accessDurationInMs, boolean instrumentOs, int bufferOsSize, int fetchIsSize) {
   this.containers = new ArrayList<>();
   this.accounts = new HashMap<>();
   this.accessDurationInMs = accessDurationInMs;
   this.instrumentOs = instrumentOs;
   this.bufferOsSize = bufferOsSize;
   this.fetchIsSize = fetchIsSize;

   for (StorageCredentials account : accounts) {
      try {
         CloudStorageAccount storage = new CloudStorageAccount(account,true);
         this.accounts.put(account.getAccountName(), storage.createCloudBlobClient());

         CloudBlobClient client = storage.createCloudBlobClient();
         this.containers.add(getStorageContainer(client,container));
      } catch (Exception ex) {
         throw new RuntimeException(ex);
      }
   }

   log.info("configured azure storage with {} accounts", accounts.size());
}
 

public PreviewStorageAzure(List<StorageCredentials> accounts, String container, ExecutorService executor) {
	this.containers = new ArrayList<>();
	this.accounts = new HashMap<>();
	this.executor = executor;

	for (StorageCredentials account : accounts) {
		try {
			CloudStorageAccount storage = new CloudStorageAccount(account, true);
			this.accounts.put(account.getAccountName(), storage.createCloudBlobClient());

			CloudBlobClient client = storage.createCloudBlobClient();
			this.containers.add(getStorageContainer(client, container));
		} catch (Exception ex) {
			throw new RuntimeException(ex);
		}
	}

	log.info("configured azure storage with {} accounts", accounts.size());
}
 

@Override
public CloudStorageAccount getCloudStorageAccount() throws InvalidKeyException, URISyntaxException {

    StringBuilder connectionString = new StringBuilder();
    connectionString.append("DefaultEndpointsProtocol=").append(protocol) //
            .append(";AccountName=").append(accountName) //
            .append(";AccountKey=").append(accountKey);

    return CloudStorageAccount.parse(connectionString.toString());
}
 

/**
 * Creates a test account that goes against the storage emulator.
 * 
 * @return The test account, or null if the emulator isn't setup.
 */
public static AzureBlobStorageTestAccount createForEmulator()
    throws Exception {
  saveMetricsConfigFile();
  NativeAzureFileSystem fs = null;
  CloudBlobContainer container = null;
  Configuration conf = createTestConfiguration();
  if (!conf.getBoolean(USE_EMULATOR_PROPERTY_NAME, false)) {
    // Not configured to test against the storage emulator.
    System.out
      .println("Skipping emulator Azure test because configuration " +
          "doesn't indicate that it's running." +
          " Please see RunningLiveWasbTests.txt for guidance.");
    return null;
  }
  CloudStorageAccount account =
      CloudStorageAccount.getDevelopmentStorageAccount();
  fs = new NativeAzureFileSystem();
  String containerName = String.format("wasbtests-%s-%tQ",
      System.getProperty("user.name"), new Date());
  container = account.createCloudBlobClient().getContainerReference(
      containerName);
  container.create();

  // Set account URI and initialize Azure file system.
  URI accountUri = createAccountUri(DEFAULT_STORAGE_EMULATOR_ACCOUNT_NAME,
      containerName);
  fs.initialize(accountUri, conf);

  // Create test account initializing the appropriate member variables.
  //
  AzureBlobStorageTestAccount testAcct =
      new AzureBlobStorageTestAccount(fs, account, container);

  return testAcct;
}
 

public AzureUploadManager(SecorConfig config) throws Exception {
    super(config);

    final String storageConnectionString =
            "DefaultEndpointsProtocol=" + mConfig.getAzureEndpointsProtocol() + ";" +
            "AccountName=" + mConfig.getAzureAccountName() + ";" +
            "AccountKey=" + mConfig.getAzureAccountKey() + ";";

    CloudStorageAccount storageAccount = CloudStorageAccount.parse(storageConnectionString);
    blobClient = storageAccount.createCloudBlobClient();
}
 

@Test
public void testCreateConnectionWithToken() throws Exception {
    String testAccountName = "someAccountName";
    AzureActiveDirectoryTokenGetter mockedTokenGetter = Mockito.mock(AzureActiveDirectoryTokenGetter.class);
    Mockito.when(mockedTokenGetter.retrieveAccessToken()).thenReturn("testToken");
    AzureConnectionWithToken sutTokenConnection = new AzureConnectionWithToken(testAccountName, mockedTokenGetter);


    CloudStorageAccount account = sutTokenConnection.getCloudStorageAccount();

    Mockito.verify(mockedTokenGetter).retrieveAccessToken();
    Assert.assertEquals(testAccountName, account.getCredentials().getAccountName());
}
 

protected String uploadPackageToAzureStorage(final File zipPackage, final CloudStorageAccount storageAccount,
                                             final String blobName) throws Exception {
    logInfo("");
    logInfo(UPLOAD_PACKAGE_START);
    final String packageUri = AzureStorageHelper.uploadFileAsBlob(zipPackage, storageAccount,
            DEPLOYMENT_PACKAGE_CONTAINER, blobName);
    logInfo(UPLOAD_PACKAGE_DONE + packageUri);
    return packageUri;
}
 

/**
 * Create CloudBlobClient instance.
 * @param flowFile An incoming FlowFile can be used for NiFi Expression Language evaluation to derive
 *                 Account Name, Account Key or SAS Token. This can be null if not available.
 */
public static CloudBlobClient createCloudBlobClient(ProcessContext context, ComponentLog logger, FlowFile flowFile) throws URISyntaxException {
    final AzureStorageCredentialsDetails storageCredentialsDetails = getStorageCredentialsDetails(context, flowFile);
    final CloudStorageAccount cloudStorageAccount = new CloudStorageAccount(
        storageCredentialsDetails.getStorageCredentials(),
        true,
        storageCredentialsDetails.getStorageSuffix(),
        storageCredentialsDetails.getStorageAccountName());
    final CloudBlobClient cloudBlobClient = cloudStorageAccount.createCloudBlobClient();

    return cloudBlobClient;
}
 

static CloudStorageAccount createTestAccount(Configuration conf)
    throws URISyntaxException, KeyProviderException {
  String testAccountName = conf.get(TEST_ACCOUNT_NAME_PROPERTY_NAME);
  if (testAccountName == null) {
    System.out
      .println("Skipping live Azure test because of missing test account." +
               " Please see RunningLiveWasbTests.txt for guidance.");
    return null;
  }
  return createStorageAccount(testAccountName, conf, false);
}
 

@Override
public List<NamedThing> getSchemaNames(RuntimeContainer container) throws IOException {
    List<NamedThing> result = new ArrayList<>();
    try {
        CloudStorageAccount storageAccount = getAzureConnection(container).getCloudStorageAccount();
        CloudBlobClient client = storageAccount.createCloudBlobClient();
        for (CloudBlobContainer c : client.listContainers()) {
            result.add(new SimpleNamedThing(c.getName(), c.getName()));
        }
    } catch (InvalidKeyException | URISyntaxException e) {
        throw new ComponentException(e);
    }
    return result;
}
 

public AzureStorageService(String connectionString, String containerName) {
  this.containerName = containerName;
  try {
    this.storageAccount = CloudStorageAccount.parse(connectionString);
  } catch (Exception e) {
    // Log the exception
    this.storageAccount = null;
  }
}
 

@Override
public CloudStorageAccount getCloudStorageAccount() {
    try {
        String token = tokenGetter.retrieveAccessToken();

        StorageCredentials credentials = new StorageCredentialsToken(accountName, token);
        return new CloudStorageAccount(credentials, true);
    } catch (Exception e) {
        throw new RuntimeException(e);
    }
}
 

private AzureBlobStorageTestAccount(NativeAzureFileSystem fs,
    CloudStorageAccount account,
    CloudBlobContainer container) {
  this.account = account;
  this.container = container;
  this.fs = fs;
}
 

/**
 * Creates a test account that goes against the storage emulator.
 * 
 * @return The test account, or null if the emulator isn't setup.
 */
public static AzureBlobStorageTestAccount createForEmulator()
    throws Exception {
  saveMetricsConfigFile();
  NativeAzureFileSystem fs = null;
  CloudBlobContainer container = null;
  Configuration conf = createTestConfiguration();
  if (!conf.getBoolean(USE_EMULATOR_PROPERTY_NAME, false)) {
    // Not configured to test against the storage emulator.
    System.out
      .println("Skipping emulator Azure test because configuration " +
          "doesn't indicate that it's running." +
          " Please see RunningLiveWasbTests.txt for guidance.");
    return null;
  }
  CloudStorageAccount account =
      CloudStorageAccount.getDevelopmentStorageAccount();
  fs = new NativeAzureFileSystem();
  String containerName = String.format("wasbtests-%s-%tQ",
      System.getProperty("user.name"), new Date());
  container = account.createCloudBlobClient().getContainerReference(
      containerName);
  container.create();

  // Set account URI and initialize Azure file system.
  URI accountUri = createAccountUri(DEFAULT_STORAGE_EMULATOR_ACCOUNT_NAME,
      containerName);
  fs.initialize(accountUri, conf);

  // Create test account initializing the appropriate member variables.
  //
  AzureBlobStorageTestAccount testAcct =
      new AzureBlobStorageTestAccount(fs, account, container);

  return testAcct;
}
 

static CloudStorageAccount createTestAccount(Configuration conf)
    throws URISyntaxException, KeyProviderException {
  String testAccountName = conf.get(TEST_ACCOUNT_NAME_PROPERTY_NAME);
  if (testAccountName == null) {
    System.out
      .println("Skipping live Azure test because of missing test account." +
               " Please see RunningLiveWasbTests.txt for guidance.");
    return null;
  }
  return createStorageAccount(testAccountName, conf, false);
}
 

@Before
public void setUp() throws Exception {
  tableClient = CloudStorageAccount.parse("DefaultEndpointsProtocol=http;" +
    "AccountName=;" +
    "AccountKey=")
                                   .createCloudTableClient();
}