Java Code Examples for org.kitesdk.data.View#deleteAll()

The following examples show how to use org.kitesdk.data.View#deleteAll() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: StagingToPersistent.java    From kite-examples with Apache License 2.0 5 votes vote down vote up 
@Override
public int run(String[] args) throws Exception {
  final long startOfToday = startOfDay();

  // the destination dataset
  Dataset<Record> persistent = Datasets.load(
      "dataset:file:/tmp/data/logs", Record.class);

  // the source: anything before today in the staging area
  Dataset<Record> staging = Datasets.load(
      "dataset:file:/tmp/data/logs_staging", Record.class);
  View<Record> ready = staging.toBefore("timestamp", startOfToday);

  ReadableSource<Record> source = CrunchDatasets.asSource(ready);

  PCollection<Record> stagedLogs = read(source);

  getPipeline().write(stagedLogs,
      CrunchDatasets.asTarget(persistent), Target.WriteMode.APPEND);

  PipelineResult result = run();

  if (result.succeeded()) {
    // remove the source data partition from staging
    ready.deleteAll();
    return 0;
  } else {
    return 1;
  }
}
Example 2
Source File: DeleteCommand.java    From kite with Apache License 2.0 5 votes vote down vote up 
@Override
public int run() throws IOException {
  if (targets == null || targets.isEmpty()) {
    throw new IllegalArgumentException("No views or datasets were specified.");
  }

  for (String uriOrName : targets) {
    if (isViewUri(uriOrName)) {
      View view = Datasets.load(uriOrName);
      Preconditions.checkArgument(viewMatches(view.getUri(), uriOrName),
          "Resolved view does not match requested view: " + view.getUri());
      if(skipTrash) {
        view.deleteAll();
      }else {
        view.moveToTrash();
      }
    } else if (isDatasetUri(uriOrName)) {
      if(skipTrash) {
        Datasets.delete(uriOrName);
      }else {
        Datasets.moveToTrash(uriOrName);
      }
    } else {
      if(skipTrash){
        getDatasetRepository().delete(namespace, uriOrName);
      }else {
        getDatasetRepository().moveToTrash(namespace, uriOrName);
      }
    }
    console.debug("Deleted {}", uriOrName);
  }

  return 0;
}
Example 3
Source File: DatasetTarget.java    From kite with Apache License 2.0 5 votes vote down vote up 
private void delete(View view) {
  try {
    boolean deleted = view.deleteAll();
    if (!deleted) {
      LOG.warn("No data was deleted.");
    }
  } catch (UnsupportedOperationException e) {
    LOG.error("Dataset view " + view + " cannot be deleted!");
    throw new CrunchRuntimeException("Dataset view cannot be deleted:" + view, e);
  }
}
Example 4
Source File: DatasetKeyOutputFormat.java    From kite with Apache License 2.0 5 votes vote down vote up 
@Override
public void checkOutputSpecs(JobContext jobContext) {
  // The committer setup will fail if the output dataset does not exist
  View<E> target = load(jobContext);
  Configuration conf = Hadoop.JobContext.getConfiguration.invoke(jobContext);
  switch (conf.getEnum(KITE_WRITE_MODE, WriteMode.DEFAULT)) {
    case APPEND:
      break;
    case OVERWRITE:
      // if the merge won't use replace, then delete the existing data
      if (!canReplace(target)) {
        target.deleteAll();
      }
      break;
    default:
    case DEFAULT:
      boolean isReady = false;
      if (target instanceof Signalable) {
        isReady = ((Signalable)target).isReady();
      }
      if (isReady || !target.isEmpty()) {
        throw new DatasetException(
            "View is not empty or has been signaled as ready: " + target);
      }
      break;
  }
}