Java Code Examples for org.kitesdk.data.spi.DefaultConfiguration#set()

@BeforeClass
public static void setConfiguration() throws Exception {
  HBaseTestUtils.getMiniCluster();

  original = DefaultConfiguration.get();
  Configuration conf = HBaseTestUtils.getConf();
  DefaultConfiguration.set(conf);

  zkQuorum = conf.get(HConstants.ZOOKEEPER_QUORUM);
  zkPort = conf.get(HConstants.ZOOKEEPER_CLIENT_PORT);

  URI defaultFs = URI.create(conf.get("fs.default.name"));
  hdfsIsDefault = "hdfs".equals(defaultFs.getScheme());
  hdfsHost = defaultFs.getHost();
  hdfsPort = Integer.toString(defaultFs.getPort());
}
 

@After
public void removeDataPath() throws IOException {
  // restore configuration
  DefaultConfiguration.set(startingConf);

  if(serviceTempDir != null) {
    FileUtils.deleteDirectory(serviceTempDir);
    serviceTempDir = null;
  }
  
  if(kiteConfigPath != null) {
    fs.delete(kiteConfigPath, true);
    kiteConfigPath = null;
  }

  if(Services.get() != null) {
    Services.get().destroy();
  }

  if(startingOozieHome == null) {
    System.clearProperty("oozie.home.dir");
  } else {
    System.setProperty("oozie.home.dir", startingOozieHome);
    startingOozieHome = null;
  }
}
 

private synchronized void loadConfiguration() {
  if(Services.get() != null) {
    KiteConfigurationService kiteService = Services.get().get(KiteConfigurationService.class);
    if(kiteService != null) {
      Configuration kiteConf = kiteService.getKiteConf();
      if(kiteConf != null) {
        DefaultConfiguration.set(kiteConf);
      } else {
        // kite conf was null
        LOG.warn("Configuration for Kite not loaded, Kite configuration service config was null.");
      }
    } else {
      // service was null
      LOG.warn("Configuration for Kite not loaded, Kite configuration service was not available.");
    }
  } else {
      // services were null
      LOG.warn("Configuration for Kite not loaded, oozie services were not available.");
  }
}
 

@After
public void removeDataPath() throws IOException {
  fs.delete(new Path("target/data"), true);
  // restore configuration
  DefaultConfiguration.set(startingConf);

  if(serviceTempDir != null) {
    FileUtils.deleteDirectory(serviceTempDir);
    serviceTempDir = null;
  }
  if(Services.get() != null) {
    Services.get().destroy();
  }

  if(startingOozieHome == null) {
    System.clearProperty("oozie.home.dir");
  } else {
    System.setProperty("oozie.home.dir", startingOozieHome);
    startingOozieHome = null;
  }
}
 

@BeforeClass
public static void addCredentials() {
  original = DefaultConfiguration.get();
  Configuration conf = DefaultConfiguration.get();
  if (ID != null) {
    conf.set("fs.s3n.awsAccessKeyId", ID);
    conf.set("fs.s3n.awsSecretAccessKey", KEY);
    conf.set("fs.s3a.access.key", ID);
    conf.set("fs.s3a.secret.key", KEY);
  }
  DefaultConfiguration.set(conf);
}
 

/**
 * Starts the services in order, passing the previous service's modified
 * Configuration object to the next.
 * 
 * @throws IOException
 */
public void start() throws IOException, InterruptedException {
  for (Service service : services) {
    service.configure(serviceConfig);
    logger.info("Running Minicluster Service: "
        + service.getClass().getName());
    service.start();
    serviceConfig.setHadoopConf(service.getHadoopConf());
    // set the default configuration so that the minicluster is used
    DefaultConfiguration.set(serviceConfig.getHadoopConf());
  }
  logger.info("All Minicluster Services running.");
}
 

@Test
public void testFindsHDFS() throws Exception {
  // set the default configuration that the loader will use
  Configuration existing = DefaultConfiguration.get();
  DefaultConfiguration.set(getConfiguration());

  FileSystemDataset<GenericRecord> dataset =
      Datasets.load("dataset:hdfs:/tmp/datasets/ns/strings");
  Assert.assertNotNull("Dataset should be found", dataset);
  Assert.assertEquals("Dataset should be located in HDFS",
      "hdfs", dataset.getFileSystem().getUri().getScheme());

  // replace the original config so the other tests are not affected
  DefaultConfiguration.set(existing);
}
 

@BeforeClass
public static void createRepositoryAndTestDatasets() throws Exception {
  // set the default configuration so that HDFS is found
  existing = DefaultConfiguration.get();
  DefaultConfiguration.set(getConfiguration());
  hdfsAuth = getDFS().getUri().getAuthority();
  descriptor = new DatasetDescriptor.Builder()
      .schemaUri("resource:schema/user.avsc")
      .build();
}
 

@Test
public void testBasicUseHDFSSchema() throws Exception {
  // this test needs to set the default configuration so that the
  // DatasetDescriptor can resolve HDFS to qualify the path. otherwise,
  // the default FS is file:/ and the avsc path is not qualified, causing
  // an IOException when it tries to read the file. Setting up HDFS correctly
  // in the environment fixes the problem.
  Configuration existing = DefaultConfiguration.get();
  DefaultConfiguration.set(getConfiguration());

  String avsc = "hdfs:/tmp/schemas/hdfsUser.avsc";
  FSDataOutputStream out = getDFS()
      .create(new Path(avsc), true /* overwrite */ );
  ByteStreams.copy(Resources.getResource("test-schemas/user.avsc").openStream(), out);
  out.close();
  command.avroSchemaFile = avsc;
  command.datasets = Lists.newArrayList("users");
  command.run();

  DatasetDescriptor expectedDescriptor = new DatasetDescriptor.Builder()
      .schemaUri("resource:test-schemas/user.avsc")
      .build();

  verify(getMockRepo()).create("default", "users", expectedDescriptor);
  verify(console).debug(contains("Created"), eq("users"));

  // restore the previous Configuration
  DefaultConfiguration.set(existing);
}
 

public static int run(Logger console, Configuration conf, String... args)
    throws Exception {
  // ensure the default config is not changed by calling Main
  Configuration original = DefaultConfiguration.get();
  Main main = new Main(console);
  main.setConf(conf);
  int rc = main.run(args);
  DefaultConfiguration.set(original);
  return rc;
}
 

private static void addToConfiguration(Properties hadoopConfiguration) {
  // base the new Configuration on the current defaults
  Configuration conf = new Configuration(DefaultConfiguration.get());

  // add all of the properties as config settings
  for (String key : hadoopConfiguration.stringPropertyNames()) {
    String value = hadoopConfiguration.getProperty(key);
    conf.set(key, value);
  }

  // replace the original Configuration
  DefaultConfiguration.set(conf);

  addedConf = true;
}
 

@AfterClass
public static void restoreOriginalConf() {
  DefaultConfiguration.set(original);
}
 

@AfterClass
public static void restoreDefaultConfiguration() {
  DefaultConfiguration.set(original);
}
 

@AfterClass
public static void restoreConfiguration() {
  DefaultConfiguration.set(originalKiteConf);
}
 

@AfterClass
public static void restoreOriginalConf() {
  DefaultConfiguration.set(original);
}
 

@OnScheduled
protected void setDefaultConfiguration(ProcessContext context)
        throws IOException {
    DefaultConfiguration.set(getConfiguration(
            context.getProperty(CONF_XML_FILES).getValue()));
}
 

@AfterClass
public static void restoreConfiguration() throws Exception {
  DefaultConfiguration.set(original);
  HBaseTestUtils.util.shutdownMiniCluster();
}
 

@AfterClass
public static void restoreDefaultConfig() {
  DefaultConfiguration.set(original);
}
 

@BeforeClass
public static void replaceDefaultConfig() {
  original = DefaultConfiguration.get();
  DefaultConfiguration.set(getConfiguration());
}
 

@AfterClass
public static void resetDefaultConfiguration() {
  DefaultConfiguration.set(existing);
}