Java Code Examples for org.kitesdk.data.spi.DefaultConfiguration#get()

@Override
public DatasetRepository getFromOptions(Map<String, String> match) {
  String path = match.get("path");
  final Path root = (path == null || path.isEmpty()) ?
      new Path("/") : new Path("/", path);

  Configuration conf = DefaultConfiguration.get();
  FileSystem fs;
  try {
    fs = FileSystem.get(fileSystemURI(match), conf);
  } catch (IOException e) {
    // "Incomplete HDFS URI, no host" => add a helpful suggestion
    if (e.getMessage().startsWith("Incomplete")) {
      throw new DatasetIOException("Could not get a FileSystem: " +
          "make sure the credentials for " + match.get(URIPattern.SCHEME) +
          " URIs are configured.", e);
    }
    throw new DatasetIOException("Could not get a FileSystem", e);
  }
  return new FileSystemDatasetRepository.Builder()
      .configuration(new Configuration(conf)) // make a modifiable copy
      .rootDirectory(fs.makeQualified(root))
      .build();
}
 

@BeforeClass
public static void setConfiguration() throws Exception {
  HBaseTestUtils.getMiniCluster();

  original = DefaultConfiguration.get();
  Configuration conf = HBaseTestUtils.getConf();
  DefaultConfiguration.set(conf);

  zkQuorum = conf.get(HConstants.ZOOKEEPER_QUORUM);
  zkPort = conf.get(HConstants.ZOOKEEPER_CLIENT_PORT);

  URI defaultFs = URI.create(conf.get("fs.default.name"));
  hdfsIsDefault = "hdfs".equals(defaultFs.getScheme());
  hdfsHost = defaultFs.getHost();
  hdfsPort = Integer.toString(defaultFs.getPort());
}
 

@Test
public void testSetupJobIsIdempotent() {
  DatasetKeyOutputFormat.MergeOutputCommitter<Object> outputCommitter
    = new DatasetKeyOutputFormat.MergeOutputCommitter<Object>();

  Configuration conf = DefaultConfiguration.get();
  DatasetKeyOutputFormat.configure(conf).appendTo(outputDataset);

  JobID jobId = new JobID("jt", 42);
  JobContext context = Hadoop.JobContext.ctor.newInstance(conf, jobId);

  // setup the job
  outputCommitter.setupJob(context);

  // call setup again to simulate an ApplicationMaster restart
  outputCommitter.setupJob(context);
}
 

@Test
public void loadConfigFromHCatAccessor() throws URIHandlerException, URISyntaxException, ServiceException, IOException {
  setupKiteConfigurationService(true, true);
  URI uri = new URI("view:file:target/data/data/nomailbox?message=hello");
  uriHandler.exists(uri, null);

  Configuration defaultConf = DefaultConfiguration.get();
  Assert.assertEquals("test.value", defaultConf.get("test.property"));

  Services.get().get(KiteConfigurationService.class).getKiteConf().set("test.value", "something.else");
  // doesn't modify default config on further exist calls
  uriHandler.exists(uri, null);

  defaultConf = DefaultConfiguration.get();
  Assert.assertEquals("test.value", defaultConf.get("test.property"));
  Assert.assertEquals("something.else", Services.get().get(KiteConfigurationService.class).getKiteConf().get("test.value"));
}
 

protected static Configuration getConfiguration(String configFiles) {
    Configuration conf = DefaultConfiguration.get();

    if (configFiles == null || configFiles.isEmpty()) {
        return conf;
    }

    for (String file : COMMA.split(configFiles)) {
        // process each resource only once
        if (conf.getResource(file) == null) {
            // use Path instead of String to get the file from the FS
            conf.addResource(new Path(file));
        }
    }

    return conf;
}
 

@Before
public void setUp() throws IOException, URISyntaxException {
  this.conf = (distributed ?
      MiniDFSTest.getConfiguration() :
      new Configuration());

  this.fs = FileSystem.get(conf);

  this.testDescriptor = new DatasetDescriptor.Builder()
      .format(Formats.AVRO)
      .schema(SchemaBuilder.record("Event").fields()
          .requiredLong("timestamp")
          .requiredString("message")
          .endRecord())
      .partitionStrategy(new PartitionStrategy.Builder()
          .year("timestamp")
          .month("timestamp")
          .day("timestamp")
          .build())
      .build();

  uriHandler = new KiteURIHandler();

  startingConf = DefaultConfiguration.get();

  startingOozieHome = System.getProperty("oozie.home.dir");
}
 

@BeforeClass
public static void createRepositoryAndTestDatasets() throws Exception {
  // set the default configuration so that HDFS is found
  existing = DefaultConfiguration.get();
  DefaultConfiguration.set(getConfiguration());
  hdfsAuth = getDFS().getUri().getAuthority();
  descriptor = new DatasetDescriptor.Builder()
      .schemaUri("resource:schema/user.avsc")
      .build();
}
 

@Test
public void testFindsHDFS() throws Exception {
  // set the default configuration that the loader will use
  Configuration existing = DefaultConfiguration.get();
  DefaultConfiguration.set(getConfiguration());

  FileSystemDataset<GenericRecord> dataset =
      Datasets.load("dataset:hdfs:/tmp/datasets/ns/strings");
  Assert.assertNotNull("Dataset should be found", dataset);
  Assert.assertEquals("Dataset should be located in HDFS",
      "hdfs", dataset.getFileSystem().getUri().getScheme());

  // replace the original config so the other tests are not affected
  DefaultConfiguration.set(existing);
}
 

@Override
public DatasetRepository getFromOptions(Map<String, String> match) {
  final Path root;
  String path = match.get("path");
  if (match.containsKey("absolute")
      && Boolean.valueOf(match.get("absolute"))) {
    root = (path == null || path.isEmpty()) ? new Path("/") : new Path("/", path);
  } else {
    root = (path == null || path.isEmpty()) ? new Path(".") : new Path(path);
  }

  Configuration conf = DefaultConfiguration.get();
  FileSystem fs;
  try {
    fs = FileSystem.get(fileSystemURI(match), conf);
  } catch (IOException e) {
    // "Incomplete HDFS URI, no host" => add a helpful suggestion
    if (e.getMessage().startsWith("Incomplete")) {
      throw new DatasetIOException("Could not get a FileSystem: " +
          "make sure the default " + match.get(URIPattern.SCHEME) +
          " URI is configured.", e);
    }
    throw new DatasetIOException("Could not get a FileSystem", e);
  }
  return new FileSystemDatasetRepository.Builder()
      .configuration(new Configuration(conf)) // make a modifiable copy
      .rootDirectory(fs.makeQualified(root))
      .build();
}
 

public Builder() {
  this.properties = Maps.newHashMap();
  this.conf = DefaultConfiguration.get();
  try {
    this.defaultFS = FileSystem.get(conf).getUri();
  } catch (IOException e) {
    throw new DatasetIOException("Cannot get the default FS", e);
  }
}
 

protected Configuration getConf() {
  if (!addedConf) {
    addToConfiguration(hadoopConfiguration);
  }
  // use the default
  return DefaultConfiguration.get();
}
 

private static void addToConfiguration(Properties hadoopConfiguration) {
  // base the new Configuration on the current defaults
  Configuration conf = new Configuration(DefaultConfiguration.get());

  // add all of the properties as config settings
  for (String key : hadoopConfiguration.stringPropertyNames()) {
    String value = hadoopConfiguration.getProperty(key);
    conf.set(key, value);
  }

  // replace the original Configuration
  DefaultConfiguration.set(conf);

  addedConf = true;
}
 

public static int run(Logger console, Configuration conf, String... args)
    throws Exception {
  // ensure the default config is not changed by calling Main
  Configuration original = DefaultConfiguration.get();
  Main main = new Main(console);
  main.setConf(conf);
  int rc = main.run(args);
  DefaultConfiguration.set(original);
  return rc;
}
 

@Test
public void testBasicUseHDFSSchema() throws Exception {
  // this test needs to set the default configuration so that the
  // DatasetDescriptor can resolve HDFS to qualify the path. otherwise,
  // the default FS is file:/ and the avsc path is not qualified, causing
  // an IOException when it tries to read the file. Setting up HDFS correctly
  // in the environment fixes the problem.
  Configuration existing = DefaultConfiguration.get();
  DefaultConfiguration.set(getConfiguration());

  String avsc = "hdfs:/tmp/schemas/hdfsUser.avsc";
  FSDataOutputStream out = getDFS()
      .create(new Path(avsc), true /* overwrite */ );
  ByteStreams.copy(Resources.getResource("test-schemas/user.avsc").openStream(), out);
  out.close();
  command.avroSchemaFile = avsc;
  command.datasets = Lists.newArrayList("users");
  command.run();

  DatasetDescriptor expectedDescriptor = new DatasetDescriptor.Builder()
      .schemaUri("resource:test-schemas/user.avsc")
      .build();

  verify(getMockRepo()).create("default", "users", expectedDescriptor);
  verify(console).debug(contains("Created"), eq("users"));

  // restore the previous Configuration
  DefaultConfiguration.set(existing);
}
 

/**
 * Build an instance of the configured {@link HiveManagedDatasetRepository}.
 *
 * @since 0.9.0
 */
@SuppressWarnings("deprecation")
public DatasetRepository build() {

  if (configuration == null) {
    this.configuration = DefaultConfiguration.get();
  }

  if (rootDirectory != null) {
    return new HiveExternalDatasetRepository(configuration, rootDirectory);
  } else {
    return new HiveManagedDatasetRepository(configuration);
  }
}
 

@BeforeClass
public static void saveOriginalConf() {
  AbstractDatasetMojo.addedConf = false;
  original = DefaultConfiguration.get();
}
 

@BeforeClass
public static void saveOriginalConf() {
  AbstractDatasetMojo.addedConf = false;
  original = DefaultConfiguration.get();
}
 

@BeforeClass
public static void defaultConfiguration() {
  originalKiteConf = DefaultConfiguration.get();
}
 

@BeforeClass
public static void saveDefaultConfiguration() {
  original = DefaultConfiguration.get();
}
 

@BeforeClass
public static void replaceDefaultConfig() {
  original = DefaultConfiguration.get();
  DefaultConfiguration.set(getConfiguration());
}