com.microsoft.azure.storage.file.CloudFileShare Java Examples

public void init(ZeppelinConfiguration conf) throws IOException {
  this.conf = conf;
  user = conf.getString(ZeppelinConfiguration.ConfVars.ZEPPELIN_NOTEBOOK_AZURE_USER);
  shareName = conf.getString(ZeppelinConfiguration.ConfVars.ZEPPELIN_NOTEBOOK_AZURE_SHARE);

  try {
    CloudStorageAccount account = CloudStorageAccount.parse(
        conf.getString(ZeppelinConfiguration.ConfVars.ZEPPELIN_NOTEBOOK_AZURE_CONNECTION_STRING));
    CloudFileClient client = account.createCloudFileClient();
    CloudFileShare share = client.getShareReference(shareName);
    share.createIfNotExists();

    CloudFileDirectory userDir = StringUtils.isBlank(user) ?
        share.getRootDirectoryReference() :
        share.getRootDirectoryReference().getDirectoryReference(user);
    userDir.createIfNotExists();

    rootDir = userDir.getDirectoryReference("notebook");
    rootDir.createIfNotExists();
  } catch (Exception e) {
    throw new IOException(e);
  }
}
 

public void init(ZeppelinConfiguration conf) throws IOException {
  this.conf = conf;
  user = conf.getString(ZeppelinConfiguration.ConfVars.ZEPPELIN_NOTEBOOK_AZURE_USER);
  shareName = conf.getString(ZeppelinConfiguration.ConfVars.ZEPPELIN_NOTEBOOK_AZURE_SHARE);

  try {
    CloudStorageAccount account = CloudStorageAccount.parse(
        conf.getString(ZeppelinConfiguration.ConfVars.ZEPPELIN_NOTEBOOK_AZURE_CONNECTION_STRING));
    CloudFileClient client = account.createCloudFileClient();
    CloudFileShare share = client.getShareReference(shareName);
    share.createIfNotExists();

    CloudFileDirectory userDir = StringUtils.isBlank(user) ?
        share.getRootDirectoryReference() :
        share.getRootDirectoryReference().getDirectoryReference(user);
    userDir.createIfNotExists();

    rootDir = userDir.getDirectoryReference("notebook");
    rootDir.createIfNotExists();
  } catch (Exception e) {
    throw new IOException(e);
  }
}
 

@Test
public void testCloudStorageAccountClientUriVerify() throws URISyntaxException, StorageException {
    StorageCredentialsAccountAndKey cred = new StorageCredentialsAccountAndKey(ACCOUNT_NAME, ACCOUNT_KEY);
    CloudStorageAccount cloudStorageAccount = new CloudStorageAccount(cred, true);

    CloudBlobClient blobClient = cloudStorageAccount.createCloudBlobClient();
    CloudBlobContainer container = blobClient.getContainerReference("container1");
    assertEquals(cloudStorageAccount.getBlobEndpoint().toString() + "/container1", container.getUri().toString());

    CloudQueueClient queueClient = cloudStorageAccount.createCloudQueueClient();
    CloudQueue queue = queueClient.getQueueReference("queue1");
    assertEquals(cloudStorageAccount.getQueueEndpoint().toString() + "/queue1", queue.getUri().toString());

    CloudTableClient tableClient = cloudStorageAccount.createCloudTableClient();
    CloudTable table = tableClient.getTableReference("table1");
    assertEquals(cloudStorageAccount.getTableEndpoint().toString() + "/table1", table.getUri().toString());

    CloudFileClient fileClient = cloudStorageAccount.createCloudFileClient();
    CloudFileShare share = fileClient.getShareReference("share1");
    assertEquals(cloudStorageAccount.getFileEndpoint().toString() + "/share1", share.getUri().toString());
}
 

private Observable<Triple<String, String, String>> createSingleFileShareAsync(final CloudFileClient client, final String volumeName, final String fileShareName, final String storageAccountKey) {
    return Observable.fromCallable(new Callable<Triple<String, String, String>>() {
        @Override
        public Triple<String, String, String> call() throws Exception {
            CloudFileShare cloudFileShare = client.getShareReference(fileShareName);
            cloudFileShare.createIfNotExists();

            return Triple.of(volumeName, fileShareName, storageAccountKey);
        }
    });
}
 

@Test
@Category({ DevFabricTests.class, DevStoreTests.class, SecondaryTests.class })
public void testFileMaximumExecutionTime() throws URISyntaxException, StorageException {
    OperationContext opContext = new OperationContext();
    setDelay(opContext, 2500);
    
    opContext.getResponseReceivedEventHandler().addListener(new StorageEvent<ResponseReceivedEvent>() {

        @Override
        public void eventOccurred(ResponseReceivedEvent eventArg) {
            // Set status code to 500 to force a retry
            eventArg.getRequestResult().setStatusCode(500);
        }
    });

    // set the maximum execution time
    FileRequestOptions options = new FileRequestOptions();
    options.setMaximumExecutionTimeInMs(2000);
    options.setTimeoutIntervalInMs(1000);

    CloudFileClient fileClient = TestHelper.createCloudFileClient();
    CloudFileShare share = fileClient.getShareReference(generateRandomName("share"));

    try {
        // 1. download attributes will fail as the share does not exist
        // 2. the executor will attempt to retry as we set the status code to 500
        // 3. maximum execution time should prevent the retry from being made
        share.downloadAttributes(null, options, opContext);
        fail("Maximum execution time was reached but request did not fail.");
    }
    catch (StorageException e) {
        assertEquals(SR.MAXIMUM_EXECUTION_TIMEOUT_EXCEPTION, e.getMessage());
    }
}
 

/**
 * Start copying a file and then abort
 *
 * @throws StorageException
 * @throws URISyntaxException
 * @throws IOException
 * @throws InvalidKeyException
 * @throws InterruptedException
 */
@Test
@Category({ DevFabricTests.class, DevStoreTests.class })
public void testCopyFileAbort()
        throws StorageException, URISyntaxException, IOException, InvalidKeyException, InterruptedException {
    final int length = 128;
    final CloudFileShare share = FileTestHelper.getRandomShareReference();
    share.create();
    final CloudFile source = FileTestHelper.uploadNewFile(share, length, null);

    // Source SAS must have read permissions
    SharedAccessFilePolicy policy = new SharedAccessFilePolicy();
    policy.setPermissions(EnumSet.of(SharedAccessFilePermissions.READ));

    Calendar cal = Calendar.getInstance(Utility.UTC_ZONE);
    cal.add(Calendar.MINUTE, 5);
    policy.setSharedAccessExpiryTime(cal.getTime());
    String sasToken = source.generateSharedAccessSignature(policy, null, null);

    // Start copy and wait for completion
    final CloudBlockBlob destination = this.container.getBlockBlobReference(source.getName() + "copyed");
    StorageCredentialsSharedAccessSignature credentials = new StorageCredentialsSharedAccessSignature(sasToken);
    destination.startCopy(new CloudFile(credentials.transformUri(source.getUri())));

    try {
        destination.abortCopy(destination.getProperties().getCopyState().getCopyId());
        BlobTestHelper.waitForCopy(destination);
        fail();
    }
    catch (StorageException e) {
        if (!e.getErrorCode().contains("NoPendingCopyOperation")) {
            throw e;
        }
    }
    finally {
        share.deleteIfExists();
    }
}
 

@Override
public BatchAIWorkspace createResource(BatchAIWorkspaces workspaces) throws Exception {
    final String groupName = SdkContext.randomResourceName("rg", 10);
    final String workspaceName = SdkContext.randomResourceName("ws", 10);
    final String vnetName = SdkContext.randomResourceName("vnet", 15);
    final String saName = SdkContext.randomResourceName("cluster", 15);
    final String shareName = "myfileshare";
    final String shareMountPath = "azurefileshare";
    final String blobFileSystemPath = "myblobsystem";
    final String containerName = "mycontainer";
    final String userName = "tirekicker";
    final String subnetName = "MySubnet";
    String storageAccountKey;
    String fileShareUri;

    BatchAIWorkspace workspace = workspaces.define(workspaceName)
            .withRegion(region)
            .withNewResourceGroup(groupName)
            .create();

    if (isPlaybackMode()) {
        storageAccountKey = "dummy_key";
        fileShareUri = "dummy_uri";
    } else {
        storageAccountKey = ensureStorageAccount(storageAccounts, saName, groupName, shareName);
        String connectionString = String.format("DefaultEndpointsProtocol=https;AccountName=%s;AccountKey=%s", saName, storageAccountKey);

        CloudFileShare cloudFileShare = CloudStorageAccount.parse(String.format("DefaultEndpointsProtocol=https;AccountName=%s;AccountKey=%s;EndpointSuffix=core.windows.net",
                saName, storageAccountKey))
                .createCloudFileClient()
                .getShareReference(shareName);
        cloudFileShare.create();

        CloudStorageAccount account = CloudStorageAccount.parse(connectionString);
        CloudBlobClient cloudBlobClient = account.createCloudBlobClient();
        CloudBlobContainer container = cloudBlobClient.getContainerReference(containerName);
        container.createIfNotExists();
        fileShareUri = cloudFileShare.getStorageUri().getPrimaryUri().toString();
    }

    Network network = networks.define(vnetName)
        .withRegion(region)
        .withExistingResourceGroup(groupName)
        .withAddressSpace("192.168.0.0/16")
        .withSubnet(subnetName, "192.168.200.0/24")
        .create();

    BatchAICluster cluster = workspace.clusters().define(clusterName)
            .withVMSize(VirtualMachineSizeTypes.STANDARD_D1_V2.toString())
            .withUserName(userName)
            .withPassword("MyPassword")
            .withAutoScale(1, 1)
            .withLowPriority()
            .defineSetupTask()
                .withCommandLine("echo Hello World!")
                .withStdOutErrPath("./outputpath")
            .attach()
            .defineAzureFileShare()
                .withStorageAccountName(saName)
                .withAzureFileUrl(fileShareUri)
                .withRelativeMountPath(shareMountPath)
                .withAccountKey(storageAccountKey)
                .attach()
            .defineAzureBlobFileSystem()
                .withStorageAccountName(saName)
                .withContainerName(containerName)
                .withRelativeMountPath(blobFileSystemPath)
                .withAccountKey(storageAccountKey)
                .attach()
            .withVirtualMachineImage("microsoft-ads", "linux-data-science-vm-ubuntu", "linuxdsvmubuntu")
            .withSubnet(network.id(), subnetName)
            .withAppInsightsComponentId("appinsightsId")
            .withInstrumentationKey("appInsightsKey")
            .create();
    printBatchAICluster(cluster);
    Assert.assertEquals("resizing", cluster.allocationState().toString());
    Assert.assertEquals(userName, cluster.adminUserName());
    Assert.assertEquals(VmPriority.LOWPRIORITY, cluster.vmPriority());
    Assert.assertEquals(1, cluster.nodeSetup().mountVolumes().azureFileShares().size());
    Assert.assertEquals(shareMountPath, cluster.nodeSetup().mountVolumes().azureFileShares().get(0).relativeMountPath());
    Assert.assertEquals(1, cluster.nodeSetup().mountVolumes().azureBlobFileSystems().size());
    Assert.assertEquals(blobFileSystemPath, cluster.nodeSetup().mountVolumes().azureBlobFileSystems().get(0).relativeMountPath());
    Assert.assertEquals(network.id() + "/subnets/" + subnetName, cluster.subnet().id());
    Assert.assertEquals("appinsightsId", cluster.nodeSetup().performanceCountersSettings().appInsightsReference().component().id());
    Assert.assertEquals("linux-data-science-vm-ubuntu", cluster.virtualMachineConfiguration().imageReference().offer());
    return workspace;
}
 

/**
 * Main function which runs the actual sample.
 *
 * @param azure instance of the azure client
 * @return true if sample runs successfully
 */
public static boolean runSample(Azure azure) {
    final String rgName = SdkContext.randomResourceName("rgACI", 15);
    final String aciName = SdkContext.randomResourceName("acisample", 20);
    final String shareName = SdkContext.randomResourceName("fileshare", 20);
    final String containerImageName = "seanmckenna/aci-hellofiles";
    final String volumeMountName = "aci-helloshare";

    try {

        //=============================================================
        // Create a container group with one container instance of default CPU core count and memory size
        //   using public Docker image "seanmckenna/aci-hellofiles" which mounts the file share created previously
        //   as read/write shared container volume.

        ContainerGroup containerGroup = azure.containerGroups().define(aciName)
            .withRegion(Region.US_WEST)
            .withNewResourceGroup(rgName)
            .withLinux()
            .withPublicImageRegistryOnly()
            .withNewAzureFileShareVolume(volumeMountName, shareName)
            .defineContainerInstance(aciName)
                .withImage(containerImageName)
                .withExternalTcpPort(80)
                .withVolumeMountSetting(volumeMountName, "/aci/logs/")
                .attach()
            .withDnsPrefix(aciName)
            .create();

        Utils.print(containerGroup);

        //=============================================================
        // Check that the container instance is up and running

        // warm up
        System.out.println("Warming up " + containerGroup.ipAddress());
        Utils.curl("http://" + containerGroup.ipAddress());
        SdkContext.sleep(30000);
        System.out.println("CURLing " + containerGroup.ipAddress());
        System.out.println(Utils.curl("http://" + containerGroup.ipAddress()));

        //=============================================================
        // Check the container instance logs

        String logContent = containerGroup.getLogContent(aciName);
        System.out.format("Logs for container instance: %s\n%s", aciName, logContent);

        //=============================================================
        // List the file share content

        String storageAccountName = containerGroup.volumes().get(volumeMountName).azureFile().storageAccountName();
        StorageAccount storageAccount = azure.storageAccounts().getByResourceGroup(rgName, storageAccountName);
        CloudFileShare cloudFileShare = CloudStorageAccount.parse(String.format("DefaultEndpointsProtocol=https;AccountName=%s;AccountKey=%s;EndpointSuffix=core.windows.net",
            storageAccountName,
            storageAccount.getKeys().get(0).value()))
            .createCloudFileClient()
            .getShareReference(shareName);
        Iterable<ListFileItem> shareContent = cloudFileShare.getRootDirectoryReference().listFilesAndDirectories();

        for (ListFileItem item : shareContent) {
            System.out.format("Found shared file %s:\n", item.getUri().toString());
        }

        //=============================================================
        // Remove the container group

        azure.containerGroups().deleteById(containerGroup.id());

        return true;
    } catch (Exception f) {
        System.out.println(f.getMessage());
        f.printStackTrace();
    } finally {
        try {
            System.out.println("Deleting Resource Group: " + rgName);
            azure.resourceGroups().beginDeleteByName(rgName);
            System.out.println("Deleted Resource Group: " + rgName);
        } catch (NullPointerException npe) {
            System.out.println("Did not create any resources in Azure. No clean up is necessary");
        } catch (Exception g) {
            g.printStackTrace();
        }
    }
    return false;
}
 

@Test
@Category(SlowTests.class)
public void testCopyFileSas()
        throws InvalidKeyException, URISyntaxException, StorageException, IOException, InterruptedException {
    // Create source on server.
    final CloudFileShare share = FileTestHelper.getRandomShareReference();
    try {
        share.create();
        final CloudFile source = share.getRootDirectoryReference().getFileReference("source");

        final String data = "String data";
        source.getMetadata().put("Test", "value");
        source.uploadText(data, Constants.UTF8_CHARSET, null, null, null);

        Calendar cal = Calendar.getInstance(Utility.UTC_ZONE);
        cal.add(Calendar.MINUTE, 5);

        // Source SAS must have read permissions
        SharedAccessFilePolicy policy = new SharedAccessFilePolicy();
        policy.setPermissions(EnumSet.of(SharedAccessFilePermissions.READ));
        policy.setSharedAccessExpiryTime(cal.getTime());

        String sasToken = source.generateSharedAccessSignature(policy, null, null);

        // Get destination reference
        final CloudBlockBlob destination = this.container.getBlockBlobReference("destination");

        // Start copy and wait for completion
        StorageCredentialsSharedAccessSignature credentials = new StorageCredentialsSharedAccessSignature(sasToken);
        String copyId = destination.startCopy(new CloudFile(credentials.transformUri(source.getUri())));
        BlobTestHelper.waitForCopy(destination);
        destination.downloadAttributes();
        assertNotNull(destination.getProperties().getEtag());

        // Check original file references for equality
        assertEquals(CopyStatus.SUCCESS, destination.getCopyState().getStatus());
        assertEquals(source.getServiceClient().getCredentials().transformUri(source.getUri()).getPath(),
                destination.getCopyState().getSource().getPath());
        assertEquals(data.length(), destination.getCopyState().getTotalBytes().intValue());
        assertEquals(data.length(), destination.getCopyState().getBytesCopied().intValue());
        assertEquals(copyId, destination.getProperties().getCopyState().getCopyId());

        // Attempt to abort the completed copy operation.
        try {
            destination.abortCopy(destination.getCopyState().getCopyId());
            fail();
        }
        catch (StorageException ex) {
            assertEquals(HttpURLConnection.HTTP_CONFLICT, ex.getHttpStatusCode());
        }

        String copyData = destination.downloadText(Constants.UTF8_CHARSET, null, null, null);
        assertEquals(data, copyData);

        source.downloadAttributes();
        BlobProperties prop1 = destination.getProperties();
        FileProperties prop2 = source.getProperties();

        assertEquals(prop1.getCacheControl(), prop2.getCacheControl());
        assertEquals(prop1.getContentEncoding(), prop2.getContentEncoding());
        assertEquals(prop1.getContentLanguage(), prop2.getContentLanguage());
        assertEquals(prop1.getContentMD5(), prop2.getContentMD5());
        assertEquals(prop1.getContentType(), prop2.getContentType());

        assertEquals("value", destination.getMetadata().get("Test"));
        assertEquals(1, destination.getMetadata().size());
    }
    finally {
        share.deleteIfExists();
    }
}
 

@Test
@Category({ DevFabricTests.class, DevStoreTests.class, SlowTests.class })
public void testCopyFileWithMetadataOverride()
        throws URISyntaxException, StorageException, IOException, InterruptedException, InvalidKeyException {
    Calendar calendar = Calendar.getInstance(Utility.UTC_ZONE);
    String data = "String data";

    final CloudFileShare share = FileTestHelper.getRandomShareReference();
    try {
        share.create();
        final CloudFile source = share.getRootDirectoryReference().getFileReference("source");
        FileTestHelper.setFileProperties(source);

        // do this to make sure the set MD5 can be compared, otherwise when the dummy value
        // doesn't match the actual MD5 an exception would be thrown
        BlobRequestOptions options = new BlobRequestOptions();
        options.setDisableContentMD5Validation(true);

        source.getMetadata().put("Test", "value");
        source.uploadText(data);

        calendar.add(Calendar.MINUTE, 5);

        // Source SAS must have read permissions
        SharedAccessFilePolicy policy = new SharedAccessFilePolicy();
        policy.setPermissions(EnumSet.of(SharedAccessFilePermissions.READ));
        policy.setSharedAccessExpiryTime(calendar.getTime());

        String sasToken = source.generateSharedAccessSignature(policy, null, null);

        // Get source BlockBlob reference
        StorageCredentialsSharedAccessSignature credentials = new StorageCredentialsSharedAccessSignature(sasToken);
        CloudBlockBlob destination = this.container.getBlockBlobReference("copy");

        destination.getMetadata().put("Test2", "value2");
        String copyId = destination.startCopy(
                FileTestHelper.defiddler(new CloudFile(credentials.transformUri(source.getUri()))));
        BlobTestHelper.waitForCopy(destination);
        destination.downloadAttributes();

        assertEquals(CopyStatus.SUCCESS, destination.getCopyState().getStatus());
        assertEquals(source.getServiceClient().getCredentials().transformUri(source.getUri()).getPath(),
                destination.getCopyState().getSource().getPath());
        assertEquals(data.length(), destination.getCopyState().getTotalBytes().intValue());
        assertEquals(data.length(), destination.getCopyState().getBytesCopied().intValue());
        assertEquals(copyId, destination.getCopyState().getCopyId());
        assertTrue(0 < destination.getCopyState().getCompletionTime().compareTo(
                new Date(calendar.get(Calendar.MINUTE) - 6)));

        String copyData = destination.downloadText(Constants.UTF8_CHARSET, null, options, null);
        assertEquals(data, copyData);

        source.downloadAttributes();
        BlobProperties prop1 = destination.getProperties();
        FileProperties prop2 = source.getProperties();

        assertEquals(prop1.getCacheControl(), prop2.getCacheControl());
        assertEquals(prop1.getContentEncoding(), prop2.getContentEncoding());
        assertEquals(prop1.getContentDisposition(),
                prop2.getContentDisposition());
        assertEquals(prop1.getContentLanguage(), prop2.getContentLanguage());
        assertEquals(prop1.getContentMD5(), prop2.getContentMD5());
        assertEquals(prop1.getContentType(), prop2.getContentType());

        assertEquals("value2", destination.getMetadata().get("Test2"));
        assertFalse(destination.getMetadata().containsKey("Test"));
        assertEquals(1, destination.getMetadata().size());
    }
    finally {
        share.deleteIfExists();
    }
}