Java Code Examples for org.apache.hadoop.conf.Configuration#setEnum()

private Configuration getConfig() {
    Configuration conf = new Configuration(false);
    String pre = "dt.operator.kafkaIn.prop.";
    conf.setEnum(pre + "initialOffset",
                 AbstractKafkaInputOperator.InitialOffset.EARLIEST);
    conf.setInt(pre + "initialPartitionCount", 1);
    conf.set(   pre + "topics",                TOPIC);
    conf.set(   pre + "clusters",              BROKER);

    pre = "dt.operator.fileOut.prop.";
    conf.set(   pre + "filePath",        FILE_DIR);
    conf.set(   pre + "baseName",        FILE_NAME);
    conf.setInt(pre + "maxLength",       40);
    conf.setInt(pre + "rotationWindows", 3);

    return conf;
}
 

private Configuration getConfig()
{
  Configuration conf = new Configuration(false);
  String pre = "dt.operator.kafkaIn.prop.";
  conf.setEnum(pre + "initialOffset", AbstractKafkaInputOperator.InitialOffset.EARLIEST);
  conf.setInt(pre + "initialPartitionCount", 1);
  conf.set(pre + "topics", TOPIC);
  conf.set(pre + "clusters", BROKER);

  pre = "dt.operator.fileOut.prop.";
  conf.set(pre + "filePath", FILE_DIR);
  conf.set(pre + "baseName", FILE_NAME);
  conf.setInt(pre + "maxLength", 40);
  conf.setInt(pre + "rotationWindows", 3);

  return conf;
}
 

private HistoryEventHandler createHandler(HistoryLogLevel logLevel) {
  Configuration conf = new Configuration(baseConfig);
  conf.setBoolean(TezConfiguration.DAG_RECOVERY_ENABLED, false);
  conf.set(TezConfiguration.TEZ_HISTORY_LOGGING_SERVICE_CLASS,
      InMemoryHistoryLoggingService.class.getName());
  if (logLevel != null) {
    conf.setEnum(TezConfiguration.TEZ_HISTORY_LOGGING_LOGLEVEL, logLevel);
  }

  DAG dag = mock(DAG.class);
  when(dag.getConf()).thenReturn(conf);

  AppContext appContext = mock(AppContext.class);
  when(appContext.getApplicationID()).thenReturn(appId);
  when(appContext.getHadoopShim()).thenReturn(new HadoopShim() {});
  when(appContext.getAMConf()).thenReturn(conf);
  when(appContext.getCurrentDAG()).thenReturn(dag);

  HistoryEventHandler handler =  new HistoryEventHandler(appContext);
  handler.init(conf);

  return handler;
}
 

public ConfigBuilder(Configuration conf) {
  this.conf = conf;
  // defaults
  conf.setEnum(IN_MEMORY_DATA_MODEL, InMemoryDataModel.GENERIC);
  conf.setBoolean(SKIP_RESIDUAL_FILTERING, false);
  conf.setBoolean(CASE_SENSITIVE, true);
  conf.setBoolean(REUSE_CONTAINERS, false);
  conf.setBoolean(LOCALITY, false);
}
 

@Test
public void hflush_syncsEverything() throws Exception {
  URI path = gcsFsIHelper.getUniqueObjectUri("hflush_syncsEverything");
  Path hadoopPath = new Path(path);

  Configuration config = getTestConfig();
  config.setEnum(GCS_OUTPUT_STREAM_TYPE.getKey(), OutputStreamType.FLUSHABLE_COMPOSITE);
  FileSystem ghfs = GoogleHadoopFileSystemIntegrationHelper.createGhfs(path, config);

  byte[] testData = new byte[5];
  new Random().nextBytes(testData);

  try (FSDataOutputStream out = ghfs.create(hadoopPath)) {
    for (int i = 0; i < testData.length; i++) {
      out.write(testData[i]);
      out.hflush();

      // Validate partly composed data always just contain the first byte because only the
      // first hflush() succeeds and all subsequent hflush() calls should be rate limited.
      int composedLength = i + 1;
      assertThat(ghfs.getFileStatus(hadoopPath).getLen()).isEqualTo(composedLength);
      assertThat(gcsFsIHelper.readFile(path)).isEqualTo(Arrays.copyOf(testData, composedLength));
    }
  }

  // Assert that data was fully written after close
  assertThat(ghfs.getFileStatus(hadoopPath).getLen()).isEqualTo(testData.length);
  assertThat(gcsFsIHelper.readFile(path)).isEqualTo(testData);
}
 

@Test
public void hflush_rateLimited_writesEverything() throws Exception {
  URI path = gcsFsIHelper.getUniqueObjectUri("hflush_syncsEverything");
  Path hadoopPath = new Path(path);

  Configuration config = getTestConfig();
  config.setEnum(GCS_OUTPUT_STREAM_TYPE.getKey(), OutputStreamType.FLUSHABLE_COMPOSITE);
  config.setLong(GCS_OUTPUT_STREAM_SYNC_MIN_INTERVAL_MS.getKey(), Duration.ofDays(1).toMillis());
  FileSystem ghfs = GoogleHadoopFileSystemIntegrationHelper.createGhfs(path, config);

  byte[] testData = new byte[10];
  new Random().nextBytes(testData);

  try (FSDataOutputStream out = ghfs.create(hadoopPath)) {
    for (byte testDataByte : testData) {
      out.write(testDataByte);
      out.hflush();

      // Validate partly composed data always just contain the first byte because only the
      // first hflush() succeeds and all subsequent hflush() calls should be rate limited.
      assertThat(ghfs.getFileStatus(hadoopPath).getLen()).isEqualTo(1);
      assertThat(gcsFsIHelper.readFile(path)).isEqualTo(new byte[] {testData[0]});
    }
  }

  // Assert that data was fully written after close
  assertThat(ghfs.getFileStatus(hadoopPath).getLen()).isEqualTo(testData.length);
  assertThat(gcsFsIHelper.readFile(path)).isEqualTo(testData);
}
 

/**
 * Submit application to the cluster and return the app id.
 * Sets the context class loader for application dependencies.
 *
 * @param appConfig
 * @return ApplicationId
 * @throws Exception
 */
public ApplicationId launchApp(AppFactory appConfig) throws Exception
{
  loadDependencies();
  Configuration conf = propertiesBuilder.conf;
  conf.setEnum(StreamingApplication.ENVIRONMENT, StreamingApplication.Environment.CLUSTER);
  LogicalPlan dag = appConfig.createApp(propertiesBuilder);
  if (UserGroupInformation.isSecurityEnabled()) {
    long hdfsTokenMaxLifeTime = conf.getLong(StramClientUtils.DT_HDFS_TOKEN_MAX_LIFE_TIME, conf.getLong(StramClientUtils.HDFS_TOKEN_MAX_LIFE_TIME, StramClientUtils.DELEGATION_TOKEN_MAX_LIFETIME_DEFAULT));
    dag.setAttribute(LogicalPlan.HDFS_TOKEN_LIFE_TIME, hdfsTokenMaxLifeTime);
    long rmTokenMaxLifeTime = conf.getLong(StramClientUtils.DT_RM_TOKEN_MAX_LIFE_TIME, conf.getLong(YarnConfiguration.RM_DELEGATION_TOKEN_MAX_LIFETIME_KEY, YarnConfiguration.RM_DELEGATION_TOKEN_MAX_LIFETIME_DEFAULT));
    dag.setAttribute(LogicalPlan.RM_TOKEN_LIFE_TIME, rmTokenMaxLifeTime);
    setTokenRefreshCredentials(dag, conf);
  }
  String tokenRefreshFactor = conf.get(StramClientUtils.TOKEN_ANTICIPATORY_REFRESH_FACTOR);
  if (tokenRefreshFactor != null && tokenRefreshFactor.trim().length() > 0) {
    dag.setAttribute(LogicalPlan.TOKEN_REFRESH_ANTICIPATORY_FACTOR, Double.parseDouble(tokenRefreshFactor));
  }
  StramClient client = new StramClient(conf, dag);
  try {
    client.start();
    LinkedHashSet<String> libjars = Sets.newLinkedHashSet();
    String libjarsCsv = conf.get(LIBJARS_CONF_KEY_NAME);
    if (libjarsCsv != null) {
      String[] jars = StringUtils.splitByWholeSeparator(libjarsCsv, StramClient.LIB_JARS_SEP);
      libjars.addAll(Arrays.asList(jars));
    }
    if (deployJars != null) {
      for (File deployJar : deployJars) {
        libjars.add(deployJar.getAbsolutePath());
      }
    }

    client.setResources(libjars);
    client.setFiles(conf.get(FILES_CONF_KEY_NAME));
    client.setArchives(conf.get(ARCHIVES_CONF_KEY_NAME));
    client.setOriginalAppId(conf.get(ORIGINAL_APP_ID));
    client.setQueueName(conf.get(QUEUE_NAME));
    String tags = conf.get(TAGS);
    if (tags != null) {
      for (String tag : tags.split(",")) {
        client.addTag(tag.trim());
      }
    }
    client.startApplication();
    return client.getApplicationReport().getApplicationId();
  } finally {
    client.stop();
  }
}
 

protected void doSubmission(String jobCreatorName, boolean defaultOutputPath)
        throws Exception {
  final Path in = new Path("foo").makeQualified(
          GridmixTestUtils.dfs.getUri(),
          GridmixTestUtils.dfs.getWorkingDirectory());
  final Path out = GridmixTestUtils.DEST.makeQualified(
          GridmixTestUtils.dfs.getUri(),
          GridmixTestUtils.dfs.getWorkingDirectory());
  final Path root = new Path(workspace.getName()).makeQualified(
      GridmixTestUtils.dfs.getUri(), GridmixTestUtils.dfs.getWorkingDirectory());
  if (!workspace.exists()) {
    assertTrue(workspace.mkdirs());
  }
  Configuration conf = null;

  try {
    ArrayList<String> argsList = new ArrayList<String>();

    argsList.add("-D" + FilePool.GRIDMIX_MIN_FILE + "=0");
    argsList.add("-D" + Gridmix.GRIDMIX_USR_RSV + "="
            + EchoUserResolver.class.getName());
    if (jobCreatorName != null) {
      argsList.add("-D" + JobCreator.GRIDMIX_JOB_TYPE + "=" + jobCreatorName);
    }

    // Set the config property gridmix.output.directory only if
    // defaultOutputPath is false. If defaultOutputPath is true, then
    // let us allow gridmix to use the path foo/gridmix/ as output dir.
    if (!defaultOutputPath) {
      argsList.add("-D" + Gridmix.GRIDMIX_OUT_DIR + "=" + out);
    }
    argsList.add("-generate");
    argsList.add(String.valueOf(GENDATA) + "m");
    argsList.add(in.toString());
    argsList.add("-"); // ignored by DebugGridmix

    String[] argv = argsList.toArray(new String[argsList.size()]);

    DebugGridmix client = new DebugGridmix();
    conf = GridmixTestUtils.mrvl.getConfig();

    CompressionEmulationUtil.setCompressionEmulationEnabled(conf, true);
    conf.setEnum(GridmixJobSubmissionPolicy.JOB_SUBMISSION_POLICY, policy);

    conf.setBoolean(GridmixJob.GRIDMIX_USE_QUEUE_IN_TRACE, true);
    UserGroupInformation ugi = UserGroupInformation.getLoginUser();
    conf.set(MRJobConfig.USER_NAME, ugi.getUserName());

    // allow synthetic users to create home directories
    GridmixTestUtils.dfs.mkdirs(root, new FsPermission((short) 777));
    GridmixTestUtils.dfs.setPermission(root, new FsPermission((short) 777));

    int res = ToolRunner.run(conf, client, argv);
    assertEquals("Client exited with nonzero status", 0, res);
    client.checkMonitor();
  } catch (Exception e) {
    e.printStackTrace();
  } finally {
    in.getFileSystem(conf).delete(in, true);
    out.getFileSystem(conf).delete(out, true);
    root.getFileSystem(conf).delete(root, true);
  }
}
 

public static void setCompressionStrategy(Configuration conf,
    CompressionStrategy strategy) {
  conf.setEnum("zlib.compress.strategy", strategy);
}
 

public static void setCompressionLevel(Configuration conf,
    CompressionLevel level) {
  conf.setEnum("zlib.compress.level", level);
}
 

protected void doSubmission(String jobCreatorName, boolean defaultOutputPath)
        throws Exception {
  final Path in = new Path("foo").makeQualified(
          GridmixTestUtils.dfs.getUri(),
          GridmixTestUtils.dfs.getWorkingDirectory());
  final Path out = GridmixTestUtils.DEST.makeQualified(
          GridmixTestUtils.dfs.getUri(),
          GridmixTestUtils.dfs.getWorkingDirectory());
  final Path root = new Path(workspace.getName()).makeQualified(
      GridmixTestUtils.dfs.getUri(), GridmixTestUtils.dfs.getWorkingDirectory());
  if (!workspace.exists()) {
    assertTrue(workspace.mkdirs());
  }
  Configuration conf = null;

  try {
    ArrayList<String> argsList = new ArrayList<String>();

    argsList.add("-D" + FilePool.GRIDMIX_MIN_FILE + "=0");
    argsList.add("-D" + Gridmix.GRIDMIX_USR_RSV + "="
            + EchoUserResolver.class.getName());
    if (jobCreatorName != null) {
      argsList.add("-D" + JobCreator.GRIDMIX_JOB_TYPE + "=" + jobCreatorName);
    }

    // Set the config property gridmix.output.directory only if
    // defaultOutputPath is false. If defaultOutputPath is true, then
    // let us allow gridmix to use the path foo/gridmix/ as output dir.
    if (!defaultOutputPath) {
      argsList.add("-D" + Gridmix.GRIDMIX_OUT_DIR + "=" + out);
    }
    argsList.add("-generate");
    argsList.add(String.valueOf(GENDATA) + "m");
    argsList.add(in.toString());
    argsList.add("-"); // ignored by DebugGridmix

    String[] argv = argsList.toArray(new String[argsList.size()]);

    DebugGridmix client = new DebugGridmix();
    conf = GridmixTestUtils.mrvl.getConfig();

    CompressionEmulationUtil.setCompressionEmulationEnabled(conf, true);
    conf.setEnum(GridmixJobSubmissionPolicy.JOB_SUBMISSION_POLICY, policy);

    conf.setBoolean(GridmixJob.GRIDMIX_USE_QUEUE_IN_TRACE, true);
    UserGroupInformation ugi = UserGroupInformation.getLoginUser();
    conf.set(MRJobConfig.USER_NAME, ugi.getUserName());

    // allow synthetic users to create home directories
    GridmixTestUtils.dfs.mkdirs(root, new FsPermission((short) 777));
    GridmixTestUtils.dfs.setPermission(root, new FsPermission((short) 777));

    int res = ToolRunner.run(conf, client, argv);
    assertEquals("Client exited with nonzero status", 0, res);
    client.checkMonitor();
  } catch (Exception e) {
    e.printStackTrace();
  } finally {
    in.getFileSystem(conf).delete(in, true);
    out.getFileSystem(conf).delete(out, true);
    root.getFileSystem(conf).delete(root, true);
  }
}
 

public static void setCompressionStrategy(Configuration conf,
    CompressionStrategy strategy) {
  conf.setEnum("zlib.compress.strategy", strategy);
}
 

public static void setCompressionLevel(Configuration conf,
    CompressionLevel level) {
  conf.setEnum("zlib.compress.level", level);
}
 

/**
 * Submit application to the cluster and return the app id.
 * Sets the context class loader for application dependencies.
 *
 * @param appConfig
 * @return ApplicationId
 * @throws Exception
 */
public ApplicationId launchApp(AppFactory appConfig) throws Exception
{
  loadDependencies();
  Configuration conf = propertiesBuilder.conf;
  conf.setEnum(StreamingApplication.ENVIRONMENT, StreamingApplication.Environment.CLUSTER);
  LogicalPlan dag = appConfig.createApp(propertiesBuilder);
  if (UserGroupInformation.isSecurityEnabled()) {
    long hdfsTokenMaxLifeTime = conf.getLong(StramClientUtils.DT_HDFS_TOKEN_MAX_LIFE_TIME, conf.getLong(StramClientUtils.HDFS_TOKEN_MAX_LIFE_TIME, StramClientUtils.DELEGATION_TOKEN_MAX_LIFETIME_DEFAULT));
    dag.setAttribute(LogicalPlan.HDFS_TOKEN_LIFE_TIME, hdfsTokenMaxLifeTime);
    LOG.debug("HDFS token life time {}", hdfsTokenMaxLifeTime);
    long hdfsTokenRenewInterval = conf.getLong(StramClientUtils.DT_HDFS_TOKEN_RENEW_INTERVAL, conf.getLong(StramClientUtils.HDFS_TOKEN_RENEW_INTERVAL, StramClientUtils.DELEGATION_TOKEN_RENEW_INTERVAL_DEFAULT));
    dag.setAttribute(LogicalPlan.HDFS_TOKEN_RENEWAL_INTERVAL, hdfsTokenRenewInterval);
    LOG.debug("HDFS token renew interval {}", hdfsTokenRenewInterval);
    long rmTokenMaxLifeTime = conf.getLong(StramClientUtils.DT_RM_TOKEN_MAX_LIFE_TIME, conf.getLong(YarnConfiguration.DELEGATION_TOKEN_MAX_LIFETIME_KEY, YarnConfiguration.DELEGATION_TOKEN_MAX_LIFETIME_DEFAULT));
    dag.setAttribute(LogicalPlan.RM_TOKEN_LIFE_TIME, rmTokenMaxLifeTime);
    LOG.debug("RM token life time {}", rmTokenMaxLifeTime);
    long rmTokenRenewInterval = conf.getLong(StramClientUtils.DT_RM_TOKEN_RENEW_INTERVAL, conf.getLong(YarnConfiguration.DELEGATION_TOKEN_RENEW_INTERVAL_KEY, YarnConfiguration.DELEGATION_TOKEN_RENEW_INTERVAL_DEFAULT));
    dag.setAttribute(LogicalPlan.RM_TOKEN_RENEWAL_INTERVAL, rmTokenRenewInterval);
    LOG.debug("RM token renew interval {}", rmTokenRenewInterval);
    setTokenRefreshCredentials(dag, conf);
  }
  String tokenRefreshFactor = conf.get(StramClientUtils.TOKEN_ANTICIPATORY_REFRESH_FACTOR);
  if (tokenRefreshFactor != null && tokenRefreshFactor.trim().length() > 0) {
    double refreshFactor = Double.parseDouble(tokenRefreshFactor);
    dag.setAttribute(LogicalPlan.TOKEN_REFRESH_ANTICIPATORY_FACTOR, refreshFactor);
    LOG.debug("Token refresh anticipatory factor {}", refreshFactor);
  }
  StramClient client = new StramClient(conf, dag);
  try {
    client.start();
    LinkedHashSet<String> libjars = Sets.newLinkedHashSet();
    String libjarsCsv = conf.get(LIBJARS_CONF_KEY_NAME);
    if (libjarsCsv != null) {
      String[] jars = StringUtils.splitByWholeSeparator(libjarsCsv, StramClient.LIB_JARS_SEP);
      libjars.addAll(Arrays.asList(jars));
    }
    if (deployJars != null) {
      for (File deployJar : deployJars) {
        libjars.add(deployJar.getAbsolutePath());
      }
    }

    client.setResources(libjars);
    client.setFiles(conf.get(FILES_CONF_KEY_NAME));
    client.setArchives(conf.get(ARCHIVES_CONF_KEY_NAME));
    client.setOriginalAppId(conf.get(ORIGINAL_APP_ID));
    client.setQueueName(conf.get(QUEUE_NAME));
    String tags = conf.get(TAGS);
    if (tags != null) {
      for (String tag : tags.split(",")) {
        client.addTag(tag.trim());
      }
    }
    client.startApplication();
    return client.getApplicationReport().getApplicationId();
  } finally {
    client.stop();
  }
}
 

private Configuration getTestConfig() {
  Configuration conf = GoogleHadoopFileSystemIntegrationHelper.getTestConfig();
  conf.setEnum(GCS_OUTPUT_STREAM_TYPE.getKey(), outputStreamType);
  conf.setEnum(GCS_OUTPUT_STREAM_PIPE_TYPE.getKey(), pipeType);
  return conf;
}
 

private static Configuration getTestConfig() {
  Configuration conf = GoogleHadoopFileSystemIntegrationHelper.getTestConfig();
  conf.setEnum(GCS_OUTPUT_STREAM_TYPE.getKey(), OutputStreamType.SYNCABLE_COMPOSITE);
  return conf;
}