Java Code Examples for org.apache.beam.sdk.io.FileSystems#delete()

The following examples show how to use org.apache.beam.sdk.io.FileSystems#delete() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: NexmarkUtils.java    From beam with Apache License 2.0 7 votes vote down vote up 
/** Frees any resources used to make the side input available. */
public static void cleanUpSideInput(NexmarkConfiguration config) throws IOException {
  switch (config.sideInputType) {
    case DIRECT:
      break;
    case CSV:
      FileSystems.delete(
          FileSystems.match(config.sideInputUrl + "*").metadata().stream()
              .map(metadata -> metadata.resourceId())
              .collect(Collectors.toList()));
      break;
    default:
      throw new IllegalArgumentException(
          String.format(
              "Unknown type of %s clean up requested", SideInputType.class.getSimpleName()));
  }
}
Example 2
Source File: BigQueryIO.java    From beam with Apache License 2.0 6 votes vote down vote up 
@Override
void cleanup(PassThroughThenCleanup.ContextContainer c) throws Exception {
  PipelineOptions options = c.getPipelineOptions();
  BigQueryOptions bqOptions = options.as(BigQueryOptions.class);
  String jobUuid = c.getJobId();
  final String extractDestinationDir =
      resolveTempLocation(bqOptions.getTempLocation(), "BigQueryExtractTemp", jobUuid);
  final String executingProject = bqOptions.getProject();
  JobReference jobRef =
      new JobReference()
          .setProjectId(executingProject)
          .setJobId(getExtractJobId(createJobIdToken(bqOptions.getJobName(), jobUuid)));

  Job extractJob = getBigQueryServices().getJobService(bqOptions).getJob(jobRef);

  if (extractJob != null) {
    List<ResourceId> extractFiles =
        getExtractFilePaths(extractDestinationDir, extractJob);
    if (extractFiles != null && !extractFiles.isEmpty()) {
      FileSystems.delete(
          extractFiles, MoveOptions.StandardMoveOptions.IGNORE_MISSING_FILES);
    }
  }
}
Example 3
Source File: FileBasedDeadLetterQueueReconsumer.java    From DataflowTemplates with Apache License 2.0 5 votes vote down vote up 
@FinishBundle
public void cleanupFiles() {
  for (ResourceId file : filesToRemove) {
    try {
      FileSystems.delete(Collections.singleton(file));
    } catch (IOException e) {
      LOG.error("Unable to delete file {}. Exception: {}", file, e);
      failedDeletions.inc();
    }
  }
  filesToRemove.clear();
}
Example 4
Source File: WriteTables.java    From beam with Apache License 2.0 5 votes vote down vote up 
static void removeTemporaryFiles(Iterable<String> files) throws IOException {
  ImmutableList.Builder<ResourceId> fileResources = ImmutableList.builder();
  for (String file : files) {
    fileResources.add(FileSystems.matchNewResource(file, false /* isDirectory */));
  }
  FileSystems.delete(fileResources.build());
}
Example 5
Source File: FileBasedIOITHelper.java    From beam with Apache License 2.0 5 votes vote down vote up 
@ProcessElement
public void processElement(ProcessContext c) throws IOException {
  MatchResult match =
      Iterables.getOnlyElement(FileSystems.match(Collections.singletonList(c.element())));

  Set<ResourceId> resourceIds = new HashSet<>();
  for (MatchResult.Metadata metadataElem : match.metadata()) {
    resourceIds.add(metadataElem.resourceId());
  }

  FileSystems.delete(resourceIds);
}
Example 6
Source File: SnowflakeIO.java    From beam with Apache License 2.0 5 votes vote down vote up 
@ProcessElement
public void processElement(ProcessContext c) throws IOException {
  String combinedPath = stagingBucketDir + "/**";
  List<ResourceId> paths =
      FileSystems.match(combinedPath).metadata().stream()
          .map(metadata -> metadata.resourceId())
          .collect(Collectors.toList());

  FileSystems.delete(paths, MoveOptions.StandardMoveOptions.IGNORE_MISSING_FILES);
}
Example 7
Source File: FakeJobService.java    From beam with Apache License 2.0 4 votes vote down vote up 
private JobStatus runLoadJob(JobReference jobRef, JobConfigurationLoad load)
    throws InterruptedException, IOException {
  TableReference destination = load.getDestinationTable();
  TableSchema schema = load.getSchema();
  checkArgument(schema != null, "No schema specified");
  List<ResourceId> sourceFiles = filesForLoadJobs.get(jobRef.getProjectId(), jobRef.getJobId());
  WriteDisposition writeDisposition = WriteDisposition.valueOf(load.getWriteDisposition());
  CreateDisposition createDisposition = CreateDisposition.valueOf(load.getCreateDisposition());

  Table existingTable = datasetService.getTable(destination);
  if (!validateDispositions(existingTable, createDisposition, writeDisposition)) {
    return new JobStatus().setState("FAILED").setErrorResult(new ErrorProto());
  }
  if (existingTable == null) {
    TableReference strippedDestination =
        destination
            .clone()
            .setTableId(BigQueryHelpers.stripPartitionDecorator(destination.getTableId()));
    existingTable = new Table().setTableReference(strippedDestination).setSchema(schema);
    if (load.getTimePartitioning() != null) {
      existingTable = existingTable.setTimePartitioning(load.getTimePartitioning());
    }
    if (load.getClustering() != null) {
      existingTable = existingTable.setClustering(load.getClustering());
    }
    datasetService.createTable(existingTable);
  }

  List<TableRow> rows = Lists.newArrayList();
  for (ResourceId filename : sourceFiles) {
    if (load.getSourceFormat().equals("NEWLINE_DELIMITED_JSON")) {
      rows.addAll(readJsonTableRows(filename.toString()));
    } else if (load.getSourceFormat().equals("AVRO")) {
      rows.addAll(readAvroTableRows(filename.toString(), schema));
    }
  }

  datasetService.insertAll(destination, rows, null);
  FileSystems.delete(sourceFiles);
  return new JobStatus().setState("DONE");
}
Example 8
Source File: GSRoundTripRuntimeTestIT.java    From components with Apache License 2.0 4 votes vote down vote up 
@After
public void clean() throws IOException {
    //TODO the directory is not really be removed, need a way to fix it
    FileSystems.delete(Collections.singletonList(FileSystems.matchNewResource(gsPath, true)),
            MoveOptions.StandardMoveOptions.IGNORE_MISSING_FILES);
}