Java Code Examples for org.kitesdk.data.Datasets#delete()

The following examples show how to use org.kitesdk.data.Datasets#delete() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: TestFileSystemDatasets.java    From kite with Apache License 2.0 6 votes vote down vote up 
@Test
public void testDatasetNotPartitioned() {
  Datasets.delete("dataset:file:/tmp/datasets/ns/test");
  final Dataset<GenericRecord> ds = Datasets.create(
      "dataset:file:/tmp/datasets/ns/test",
      new DatasetDescriptor.Builder()
          .schema(schema)
          .build());

  Assert.assertEquals("Should work for empty relative directory",
      ds, FileSystemDatasets.viewForUri(ds, "file:/tmp/datasets/ns/test"));

  TestHelpers.assertThrows("Should reject paths in a non-partitioned dataset",
      IllegalArgumentException.class, new Runnable() {
        @Override
        public void run() {
          FileSystemDatasets.viewForUri(ds, "y=2014/m=03/d=14");
        }
      });
}
Example 2
Source File: TestFileSystemPartitionView.java    From kite with Apache License 2.0 6 votes vote down vote up 
@Before
public void createTestDatasets() {
  Datasets.delete("dataset:file:/tmp/datasets/unpartitioned");
  Datasets.delete("dataset:file:/tmp/datasets/partitioned");

  DatasetDescriptor descriptor = new DatasetDescriptor.Builder()
      .schema(TestRecord.class)
      .build();
  unpartitioned = Datasets.create("dataset:file:/tmp/datasets/unpartitioned",
      descriptor, TestRecord.class);

  descriptor = new DatasetDescriptor.Builder(descriptor)
      .partitionStrategy(new PartitionStrategy.Builder()
          .hash("id", 4)
          .build())
      .build();
  partitioned = Datasets.create("dataset:file:/tmp/datasets/partitioned",
      descriptor, TestRecord.class);

  writeTestRecords(unpartitioned);
  writeTestRecords(partitioned);
}
Example 3
Source File: TestCreateDataset.java    From kite with Apache License 2.0 6 votes vote down vote up 
@Test
public void testCreateWithRepositoryURI() throws Exception {
  try {
    CreateDatasetMojo mojo = new CreateDatasetMojo();
    mojo.hadoopConfiguration = dfsProps;
    mojo.avroSchemaFile = "schema/user.avsc";
    mojo.repositoryUri = "repo:hdfs:/tmp/data";
    mojo.datasetNamespace = "ns";
    mojo.datasetName = "users";

    mojo.execute();

    Assert.assertTrue("Dataset should exist", Datasets.exists(DATASET_URI));
  } finally {
    Datasets.delete(DATASET_URI);
  }
}
Example 4
Source File: TestKiteProcessorsCluster.java    From localization_nifi with Apache License 2.0 5 votes vote down vote up 
@Test
public void testBasicStoreToHive() throws IOException {
    String datasetUri = "dataset:hive:ns/test";

    Dataset<Record> dataset = Datasets.create(datasetUri, descriptor, Record.class);

    TestRunner runner = TestRunners.newTestRunner(StoreInKiteDataset.class);
    runner.assertNotValid();

    runner.setProperty(StoreInKiteDataset.KITE_DATASET_URI, datasetUri);
    runner.assertValid();

    List<Record> users = Lists.newArrayList(
            user("a", "[email protected]"),
            user("b", "[email protected]"),
            user("c", "[email protected]")
    );

    runner.enqueue(streamFor(users));
    runner.run();

    runner.assertAllFlowFilesTransferred("success", 1);
    List<Record> stored = Lists.newArrayList(
            (Iterable<Record>) dataset.newReader());
    Assert.assertEquals("Records should match", users, stored);

    Datasets.delete(datasetUri);
}
Example 5
Source File: KiteDatasetExecutor.java    From sqoop-on-spark with Apache License 2.0 5 votes vote down vote up 
/**
 * Merges a dataset into this.
 */
public void mergeDataset(String uri) {
  FileSystemDataset<GenericRecord> update = Datasets.load(uri);
  if (dataset instanceof FileSystemDataset) {
    ((FileSystemDataset<GenericRecord>) dataset).merge(update);
    // And let's completely drop the temporary dataset
    Datasets.delete(uri);
  } else {
    throw new SqoopException(
        KiteConnectorError.GENERIC_KITE_CONNECTOR_0000, uri);
  }
}
Example 6
Source File: TestKiteProcessorsCluster.java    From nifi with Apache License 2.0 5 votes vote down vote up 
@Test
public void testBasicStoreToHive() throws IOException {
    String datasetUri = "dataset:hive:ns/test";

    Dataset<Record> dataset = Datasets.create(datasetUri, descriptor, Record.class);

    TestRunner runner = TestRunners.newTestRunner(StoreInKiteDataset.class);
    runner.assertNotValid();

    runner.setProperty(StoreInKiteDataset.KITE_DATASET_URI, datasetUri);
    runner.assertValid();

    List<Record> users = Lists.newArrayList(
            user("a", "[email protected]"),
            user("b", "[email protected]"),
            user("c", "[email protected]")
    );

    runner.enqueue(streamFor(users));
    runner.run();

    runner.assertAllFlowFilesTransferred("success", 1);
    List<Record> stored = Lists.newArrayList(
            (Iterable<Record>) dataset.newReader());
    Assert.assertEquals("Records should match", users, stored);

    Datasets.delete(datasetUri);
}
Example 7
Source File: DeleteUserDataset.java    From kite-examples with Apache License 2.0 5 votes vote down vote up 
@Override
public int run(String[] args) throws Exception {
  // Delete the users dataset
  boolean success = Datasets.delete("dataset:hdfs:/tmp/data/users");

  return success ? 0 : 1;
}
Example 8
Source File: TestViewUris.java    From kite with Apache License 2.0 5 votes vote down vote up 
@BeforeClass
public static void createTestDataset() {
  Datasets.delete("dataset:file:/tmp/test_name");
  test = Datasets.create("dataset:file:/tmp/test_name",
      new DatasetDescriptor.Builder()
          .schema(SCHEMA)
          .partitionStrategy(STRATEGY)
          .build());
}
Example 9
Source File: TestFileSystemDatasets.java    From kite with Apache License 2.0 5 votes vote down vote up 
@Before
public void createFileSystemDataset() {
  String uri = "dataset:file:/tmp/datasets/ns/test";
  DatasetDescriptor descriptor = new DatasetDescriptor.Builder()
      .schema(schema)
      .partitionStrategy(ymd)
      .build();
  Datasets.delete(uri);
  this.dataset = Datasets.create(uri, descriptor);
}
Example 10
Source File: DeleteDataset.java    From kite-examples with Apache License 2.0 5 votes vote down vote up 
@Override
public int run(String[] args) throws Exception {

  // Drop the events dataset
  boolean success = Datasets.delete("dataset:hive:/tmp/data/default/events");

  return success ? 0 : 1;
}
Example 11
Source File: DeleteDataset.java    From kite-examples with Apache License 2.0 5 votes vote down vote up 
@Override
public int run(String[] args) throws Exception {

  // Drop the events dataset
  boolean success = Datasets.delete("dataset:hive:/tmp/data/default/events");

  return success ? 0 : 1;
}
Example 12
Source File: DeleteDatasetMojo.java    From kite with Apache License 2.0 5 votes vote down vote up 
@Override
public void execute() throws MojoExecutionException, MojoFailureException {
  getConf(); // ensure properties are added to DefaultConfig

  if (uri != null) {
    Datasets.delete(uri);
  } else {
    LOG.warn(
        "kite.datasetName is deprecated, instead use kite.uri=<dataset-uri>");
    Preconditions.checkArgument(datasetName != null,
        "kite.datasetName is required if kite.uri is not used");
    DatasetRepository repo = getDatasetRepository();
    repo.delete(datasetNamespace, datasetName);
  }
}
Example 13
Source File: TestFileSystemPartitionView.java    From kite with Apache License 2.0 4 votes vote down vote up 
@Test
public void testEscapedURIs() {
  Datasets.delete("dataset:file:/tmp/datasets/string_partitioned");

  // build a new dataset with a string partition field
  DatasetDescriptor descriptor = new DatasetDescriptor.Builder()
      .partitionStrategy(new PartitionStrategy.Builder()
          .identity("data", "d_copy")
          .build())
      .schema(TestRecord.class)
      .build();

  FileSystemDataset<TestRecord> d = Datasets.create(
      "dataset:file:/tmp/datasets/string_partitioned",
      descriptor, TestRecord.class);

  writeTestRecords(d);

  FileSystemPartitionView<TestRecord> partition = d.getPartitionView(
      URI.create("file:/tmp/datasets/string_partitioned/d_copy=test%2F-0"));
  Assert.assertEquals("Should accept escaped full URI",
      URI.create("file:/tmp/datasets/string_partitioned/d_copy=test%2F-0"),
      partition.getLocation());
  Assert.assertEquals("Should should have correctly escaped relative URI",
      URI.create("d_copy=test%2F-0"), partition.getRelativeLocation());
  Assert.assertEquals("Should have correctly escaped constraints",
      d.unbounded.getConstraints().with("d_copy", "test/-0"),
      partition.getConstraints());

  partition = d.getPartitionView(
      new Path("file:/tmp/datasets/string_partitioned/d_copy=test%2F-0"));
  Assert.assertEquals("Should accept escaped full URI",
      URI.create("file:/tmp/datasets/string_partitioned/d_copy=test%2F-0"),
      partition.getLocation());
  Assert.assertEquals("Should should have correctly escaped relative URI",
      URI.create("d_copy=test%2F-0"), partition.getRelativeLocation());
  Assert.assertEquals("Should have correctly escaped constraints",
      d.unbounded.getConstraints().with("d_copy", "test/-0"),
      partition.getConstraints());

  Datasets.delete("dataset:file:/tmp/datasets/string_partitioned");
}
Example 14
Source File: TestFileSystemPartitionView.java    From kite with Apache License 2.0 4 votes vote down vote up 
@After
public void removeTestDatasets() {
  Datasets.delete("dataset:file:/tmp/datasets/unpartitioned");
  Datasets.delete("dataset:file:/tmp/datasets/partitioned");
}
Example 15
Source File: TestS3Dataset.java    From kite with Apache License 2.0 4 votes vote down vote up 
@Test
public void testBasics3a() {
  // only run this test if credentials are present
  Assume.assumeTrue(ID != null && !ID.isEmpty());

  String uri = "dataset:s3a://" + BUCKET + "/ns/test";

  // make sure the dataset doesn't already exist
  Datasets.delete(uri);

  DatasetDescriptor descriptor = new DatasetDescriptor.Builder()
      .schemaLiteral("\"string\"")
      .build();

  Dataset<String> dataset = Datasets.create(uri, descriptor, String.class);

  List<String> expected = Lists.newArrayList("a", "b", "time");
  DatasetWriter<String> writer = null;
  try {
    writer = dataset.newWriter();
    for (String s : expected) {
      writer.write(s);
    }
  } finally {
    if (writer != null) {
      writer.close();
    }
  }

  DatasetReader<String> reader = null;
  try {
    reader = dataset.newReader();
    Assert.assertEquals("Should match written strings",
        expected, Lists.newArrayList((Iterator<String>) reader));
  } finally {
    if (reader != null) {
      reader.close();
    }
  }

  // clean up
  Datasets.delete(uri);
}
Example 16
Source File: TestPartitionReplacement.java    From kite with Apache License 2.0 4 votes vote down vote up 
@After
public void removeTestDatasets() {
  Datasets.delete("dataset:file:/tmp/datasets/unpartitioned");
  Datasets.delete("dataset:file:/tmp/datasets/partitioned");
  Datasets.delete("dataset:file:/tmp/datasets/temporary");
}
Example 17
Source File: TestKiteStorageProcessor.java    From nifi with Apache License 2.0 4 votes vote down vote up 
@After
public void deleteDataset() throws Exception {
    Datasets.delete(datasetUri);
}
Example 18
Source File: TestConfigurationProperty.java    From nifi with Apache License 2.0 4 votes vote down vote up 
@After
public void deleteDataset() throws Exception {
    Datasets.delete(datasetUri);
}
Example 19
Source File: KiteDatasetExecutor.java    From sqoop-on-spark with Apache License 2.0 4 votes vote down vote up 
/**
 * Deletes current dataset physically.
 */
public void deleteDataset() {
  Datasets.delete(dataset.getUri().toString());
}
Example 20
Source File: TestS3Dataset.java    From kite with Apache License 2.0 4 votes vote down vote up 
@Test
public void testBasics3n() {
  // only run this test if credentials are present
  Assume.assumeTrue(ID != null && !ID.isEmpty());

  String uri = "dataset:s3n://" + BUCKET + "/ns/test";

  // make sure the dataset doesn't already exist
  Datasets.delete(uri);

  DatasetDescriptor descriptor = new DatasetDescriptor.Builder()
      .schemaLiteral("\"string\"")
      .build();

  Dataset<String> dataset = Datasets.create(uri, descriptor, String.class);

  List<String> expected = Lists.newArrayList("a", "b", "time");
  DatasetWriter<String> writer = null;
  try {
    writer = dataset.newWriter();
    for (String s : expected) {
      writer.write(s);
    }
  } finally {
    if (writer != null) {
      writer.close();
    }
  }

  DatasetReader<String> reader = null;
  try {
    reader = dataset.newReader();
    Assert.assertEquals("Should match written strings",
        expected, Lists.newArrayList((Iterator<String>) reader));
  } finally {
    if (reader != null) {
      reader.close();
    }
  }

  // clean up
  Datasets.delete(uri);
}