com.google.cloud.storage.contrib.nio.CloudStorageFileSystem Java Examples

@Override
public Path toPath() {
    // special case GCS, in case the filesystem provider wasn't installed properly but is available.
    if (CloudStorageFileSystem.URI_SCHEME.equals(getURI().getScheme())) {
        final Path tempPath = BucketUtils.getPathOnGcs(getURIString());
        setCachedPath(tempPath);
        return tempPath;
    } else {
        try {
            return super.toPath();
        } catch (FileSystemNotFoundException e) {
            try {
                ClassLoader cl = Thread.currentThread().getContextClassLoader();
                if (cl == null) {
                    throw e;
                }
                return FileSystems.newFileSystem(getURI(), new HashMap<>(), cl).provider().getPath(getURI());
            } catch (final IOException ioe) {
                throw new GATKException("IOException loading new file system", ioe);
            }
        }
    }
}
 

/**
 * Note: Please use Models from zoltar-models module.
 *
 * <p>Returns a TensorFlow model with metadata given {@link SavedModelBundle} export directory URI
 * and {@link Options}.
 */
public static TensorFlowModel create(
    final Model.Id id,
    final URI modelResource,
    final Options options,
    final String signatureDefinition)
    throws IOException {
  // GCS requires that directory URIs have a trailing slash, so add the slash if it's missing
  // and the URI starts with 'gs'.
  final URI normalizedUri =
      !CloudStorageFileSystem.URI_SCHEME.equalsIgnoreCase(modelResource.getScheme())
              || modelResource.toString().endsWith("/")
          ? modelResource
          : URI.create(modelResource.toString() + "/");
  final URI localDir = FileSystemExtras.downloadIfNonLocal(normalizedUri);
  final SavedModelBundle model =
      SavedModelBundle.load(localDir.toString(), options.tags().toArray(new String[0]));
  final MetaGraphDef metaGraphDef;
  try {
    metaGraphDef = extractMetaGraphDefinition(model);
  } catch (TensorflowMetaGraphDefParsingException e) {
    throw new IOException(e);
  }
  final SignatureDef signatureDef = metaGraphDef.getSignatureDefOrThrow(signatureDefinition);

  return new AutoValue_TensorFlowModel(
      id,
      model,
      options,
      metaGraphDef,
      signatureDef,
      toNameMap(signatureDef.getInputsMap()),
      toNameMap(signatureDef.getOutputsMap()));
}
 

/**
 * String -> Path. This *should* not be necessary (use Paths.get(URI.create(...)) instead) , but it currently is
 * on Spark because using the fat, shaded jar breaks the registration of the GCS FilesystemProvider.
 * To transform other types of string URLs into Paths, use IOUtils.getPath instead.
 */
public static java.nio.file.Path getPathOnGcs(String gcsUrl) {
    // use a split limit of -1 to preserve empty split tokens, especially trailing slashes on directory names
    final String[] split = gcsUrl.split("/", -1);
    final String BUCKET = split[2];
    final String pathWithoutBucket = String.join("/", Arrays.copyOfRange(split, 3, split.length));
    return CloudStorageFileSystem.forBucket(BUCKET).getPath(pathWithoutBucket);
}
 

/**
 * Get an authenticated GCS-backed NIO FileSystem object representing the selected projected and bucket.
 * Credentials are found automatically when running on Compute/App engine, logged into gcloud, or
 * if the GOOGLE_APPLICATION_CREDENTIALS env. variable is set. In that case leave credentials null.
 * Otherwise, you must pass the contents of the service account credentials file.
 * See https://github.com/GoogleCloudPlatform/gcloud-java#authentication
 *
 * Note that most of the time it's enough to just open a file via
 * Files.newInputStream(Paths.get(URI.create( path ))).
 **/
public static java.nio.file.FileSystem getAuthenticatedGcs(String projectId, String bucket, byte[] credentials) throws IOException {
    StorageOptions.Builder builder = StorageOptions.newBuilder()
            .setProjectId(projectId);
    if (null != credentials) {
        builder = builder.setCredentials(GoogleCredentials.fromStream(new ByteArrayInputStream(credentials)));
    }
    // generous timeouts, to avoid tests failing when not warranted.
    StorageOptions storageOptions = setGenerousTimeouts(builder).build();

    // 2. Create GCS filesystem object with those credentials
    return CloudStorageFileSystem.forBucket(bucket, CloudStorageConfiguration.DEFAULT, storageOptions);
}
 

public static void main(String... args) throws IOException {
  // Create a file system for the bucket using the service account credentials
  // saved in the file below.
  String myCredentials = "/path/to/my/key.json";
  CloudStorageFileSystem fs =
      CloudStorageFileSystem.forBucket(
          "mybucket",
          CloudStorageConfiguration.DEFAULT,
          StorageOptions.newBuilder()
              .setCredentials(
                  ServiceAccountCredentials.fromStream(new FileInputStream(myCredentials)))
              .build());
  // Can now read and write to the bucket using fs
  // (see e.g. ReadAllLines for an example).
}
 

public static void main(String... args) throws IOException {
  // Create a file system for the bucket
  CloudStorageFileSystem fs = CloudStorageFileSystem.forBucket("bucket");
  byte[] data = "hello world".getBytes(StandardCharsets.UTF_8);
  Path path = fs.getPath("/object");
  // Write a file in the bucket
  Files.write(path, data);
  // Read a file from the bucket
  data = Files.readAllBytes(path);
}