Java Code Examples for org.apache.hadoop.util.StringUtils#join()

@Test
@TestException(exception = ServiceException.class, msgRegExp = "H01.*")
@TestDir
public void noKerberosKeytabProperty() throws Exception {
  String dir = TestDirHelper.getTestDir().getAbsolutePath();
  String services = StringUtils.join(",",
  Arrays.asList(InstrumentationService.class.getName(),
                SchedulerService.class.getName(),
                FileSystemAccessService.class.getName()));
  Configuration conf = new Configuration(false);
  conf.set("server.services", services);
  conf.set("server.hadoop.authentication.type", "kerberos");
  conf.set("server.hadoop.authentication.kerberos.keytab", " ");
  Server server = new Server("server", dir, dir, dir, dir, conf);
  server.init();
}
 

@Test
@TestException(exception = FileSystemAccessException.class, msgRegExp = "H05.*")
@TestDir
public void NameNodeNotinWhitelists() throws Exception {
  String dir = TestDirHelper.getTestDir().getAbsolutePath();
  String services = StringUtils.join(",",
    Arrays.asList(InstrumentationService.class.getName(),
                  SchedulerService.class.getName(),
                  FileSystemAccessService.class.getName()));
  Configuration conf = new Configuration(false);
  conf.set("server.services", services);
  conf.set("server.hadoop.name.node.whitelist", "NN");
  Server server = new Server("server", dir, dir, dir, dir, conf);
  server.init();
  FileSystemAccessService fsAccess = (FileSystemAccessService) server.get(FileSystemAccess.class);
  fsAccess.validateNamenode("NNx");
}
 

@Test
@TestDir
public void serviceHadoopConf() throws Exception {
  String dir = TestDirHelper.getTestDir().getAbsolutePath();
  String services = StringUtils.join(",",
    Arrays.asList(InstrumentationService.class.getName(),
                  SchedulerService.class.getName(),
                  FileSystemAccessService.class.getName()));
  Configuration conf = new Configuration(false);
  conf.set("server.services", services);

  Server server = new Server("server", dir, dir, dir, dir, conf);
  server.init();
  FileSystemAccessService fsAccess = (FileSystemAccessService) server.get(FileSystemAccess.class);
  Assert.assertEquals(fsAccess.serviceHadoopConf.get("foo"), "FOO");
  server.destroy();
}
 

/**
 * Concat existing files together.
 * @param f the path to the target destination.
 * @param psrcs the paths to the sources to use for the concatenation.
 *
 * @throws IOException
 */
@Override
public void concat(Path f, Path[] psrcs) throws IOException {
  List<String> strPaths = new ArrayList<String>(psrcs.length);
  for(Path psrc : psrcs) {
    strPaths.add(psrc.toUri().getPath());
  }
  String srcs = StringUtils.join(",", strPaths);

  Map<String, String> params = new HashMap<String, String>();
  params.put(OP_PARAM, Operation.CONCAT.toString());
  params.put(SOURCES_PARAM, srcs);
  HttpURLConnection conn = getConnection(Operation.CONCAT.getMethod(),
      params, f, true);
  HttpExceptionUtils.validateResponse(conn, HttpURLConnection.HTTP_OK);
}
 

private static void createShuffleHandlerFiles(File logDir, String user,
    String appId, String appAttemptId, Configuration conf,
    List<File> fileMap) throws IOException {
  String attemptDir =
      StringUtils.join(Path.SEPARATOR,
          Arrays.asList(new String[] { logDir.getAbsolutePath(),
              ContainerLocalizer.USERCACHE, user,
              ContainerLocalizer.APPCACHE, appId, "output", appAttemptId }));
  File appAttemptDir = new File(attemptDir);
  appAttemptDir.mkdirs();
  System.out.println(appAttemptDir.getAbsolutePath());
  File indexFile = new File(appAttemptDir, "file.out.index");
  fileMap.add(indexFile);
  createIndexFile(indexFile, conf);
  File mapOutputFile = new File(appAttemptDir, "file.out");
  fileMap.add(mapOutputFile);
  createMapOutputFile(mapOutputFile, conf);
}
 

@Test (timeout = 120000)
public void testSetClasspathWithJobClassloader() throws IOException {
  Configuration conf = new Configuration();
  conf.setBoolean(MRConfig.MAPREDUCE_APP_SUBMISSION_CROSS_PLATFORM, true);
  conf.setBoolean(MRJobConfig.MAPREDUCE_JOB_CLASSLOADER, true);
  Map<String, String> env = new HashMap<String, String>();
  MRApps.setClasspath(env, conf);
  String cp = env.get("CLASSPATH");
  String appCp = env.get("APP_CLASSPATH");
  assertFalse("MAPREDUCE_JOB_CLASSLOADER true, but job.jar is in the"
    + " classpath!", cp.contains("jar" + ApplicationConstants.CLASS_PATH_SEPARATOR + "job"));
  assertFalse("MAPREDUCE_JOB_CLASSLOADER true, but PWD is in the classpath!",
    cp.contains("PWD"));
  String expectedAppClasspath = StringUtils.join(ApplicationConstants.CLASS_PATH_SEPARATOR,
    Arrays.asList(ApplicationConstants.Environment.PWD.$$(), "job.jar/job.jar",
      "job.jar/classes/", "job.jar/lib/*",
      ApplicationConstants.Environment.PWD.$$() + "/*"));
  assertEquals("MAPREDUCE_JOB_CLASSLOADER true, but job.jar is not in the app"
    + " classpath!", expectedAppClasspath, appCp);
}
 

/**
 * Concat existing files together.
 * @param f the path to the target destination.
 * @param psrcs the paths to the sources to use for the concatenation.
 *
 * @throws IOException
 */
@Override
public void concat(Path f, Path[] psrcs) throws IOException {
  List<String> strPaths = new ArrayList<String>(psrcs.length);
  for(Path psrc : psrcs) {
    strPaths.add(psrc.toUri().getPath());
  }
  String srcs = StringUtils.join(",", strPaths);

  Map<String, String> params = new HashMap<String, String>();
  params.put(OP_PARAM, Operation.CONCAT.toString());
  params.put(SOURCES_PARAM, srcs);
  HttpURLConnection conn = getConnection(Operation.CONCAT.getMethod(),
      params, f, true);
  HttpExceptionUtils.validateResponse(conn, HttpURLConnection.HTTP_OK);
}
 

@Test
@TestException(exception = ServiceException.class, msgRegExp = "H02.*")
@TestDir
public void kerberosInitializationFailure() throws Exception {
  String dir = TestDirHelper.getTestDir().getAbsolutePath();
  String services = StringUtils.join(",",
    Arrays.asList(InstrumentationService.class.getName(),
                  SchedulerService.class.getName(),
                  FileSystemAccessService.class.getName()));
  Configuration conf = new Configuration(false);
  conf.set("server.services", services);
  conf.set("server.hadoop.authentication.type", "kerberos");
  conf.set("server.hadoop.authentication.kerberos.keytab", "/tmp/foo");
  conf.set("server.hadoop.authentication.kerberos.principal", "foo@FOO");
  Server server = new Server("server", dir, dir, dir, dir, conf);
  server.init();
}
 

@Test
@TestDir
public void serviceHadoopConf() throws Exception {
  String dir = TestDirHelper.getTestDir().getAbsolutePath();
  String services = StringUtils.join(",",
    Arrays.asList(InstrumentationService.class.getName(),
                  SchedulerService.class.getName(),
                  FileSystemAccessService.class.getName()));
  Configuration conf = new Configuration(false);
  conf.set("server.services", services);

  Server server = new Server("server", dir, dir, dir, dir, conf);
  server.init();
  FileSystemAccessService fsAccess = (FileSystemAccessService) server.get(FileSystemAccess.class);
  Assert.assertEquals(fsAccess.serviceHadoopConf.get("foo"), "FOO");
  server.destroy();
}
 

@Test
@TestDir
@SuppressWarnings("unchecked")
public void sampling() throws Exception {
  String dir = TestDirHelper.getTestDir().getAbsolutePath();
  String services = StringUtils.join(",", Arrays.asList(InstrumentationService.class.getName(),
                                                        SchedulerService.class.getName()));
  Configuration conf = new Configuration(false);
  conf.set("server.services", services);
  Server server = new Server("server", dir, dir, dir, dir, conf);
  server.init();
  Instrumentation instrumentation = server.get(Instrumentation.class);

  final AtomicInteger count = new AtomicInteger();

  Instrumentation.Variable<Long> varToSample = new Instrumentation.Variable<Long>() {
    @Override
    public Long getValue() {
      return (long) count.incrementAndGet();
    }
  };
  instrumentation.addSampler("g", "s", 10, varToSample);

  sleep(2000);
  int i = count.get();
  assertTrue(i > 0);

  Map<String, Map<String, ?>> snapshot = instrumentation.getSnapshot();
  Map<String, Map<String, Object>> samplers = (Map<String, Map<String, Object>>) snapshot.get("samplers");
  InstrumentationService.Sampler sampler = (InstrumentationService.Sampler) samplers.get("g").get("s");
  assertTrue(sampler.getRate() > 0);

  server.destroy();
}
 

@Test
@TestException(exception = ServiceException.class, msgRegExp = "H09.*")
@TestDir
public void invalidSecurity() throws Exception {
  String dir = TestDirHelper.getTestDir().getAbsolutePath();
  String services = StringUtils.join(",",
    Arrays.asList(InstrumentationService.class.getName(),
                  SchedulerService.class.getName(),
                  FileSystemAccessService.class.getName()));
  Configuration conf = new Configuration(false);
  conf.set("server.services", services);
  conf.set("server.hadoop.authentication.type", "foo");
  Server server = new Server("server", dir, dir, dir, dir, conf);
  server.init();
}
 

void logJobHistoryVertexFailedEvent(VertexState state) throws IOException {
  VertexFinishedEvent finishEvt = new VertexFinishedEvent(vertexId,
      vertexName, initTimeRequested, initedTime, startTimeRequested,
      startedTime, clock.getTime(), state, StringUtils.join(LINE_SEPARATOR,
          getDiagnostics()), getAllCounters(), getVertexStats());
  this.appContext.getHistoryHandler().handleCriticalEvent(
      new DAGHistoryEvent(getDAGId(), finishEvt));
}
 

@Test (timeout = 3000000)
public void testSetClasspathWithFramework() throws IOException {
  final String FRAMEWORK_NAME = "some-framework-name";
  final String FRAMEWORK_PATH = "some-framework-path#" + FRAMEWORK_NAME;
  Configuration conf = new Configuration();
  conf.setBoolean(MRConfig.MAPREDUCE_APP_SUBMISSION_CROSS_PLATFORM, true);
  conf.set(MRJobConfig.MAPREDUCE_APPLICATION_FRAMEWORK_PATH, FRAMEWORK_PATH);
  Map<String, String> env = new HashMap<String, String>();
  try {
    MRApps.setClasspath(env, conf);
    fail("Failed to catch framework path set without classpath change");
  } catch (IllegalArgumentException e) {
    assertTrue("Unexpected IllegalArgumentException",
        e.getMessage().contains("Could not locate MapReduce framework name '"
            + FRAMEWORK_NAME + "'"));
  }

  env.clear();
  final String FRAMEWORK_CLASSPATH = FRAMEWORK_NAME + "/*.jar";
  conf.set(MRJobConfig.MAPREDUCE_APPLICATION_CLASSPATH, FRAMEWORK_CLASSPATH);
  MRApps.setClasspath(env, conf);
  final String stdClasspath = StringUtils.join(ApplicationConstants.CLASS_PATH_SEPARATOR,
      Arrays.asList("job.jar/job.jar", "job.jar/classes/", "job.jar/lib/*",
          ApplicationConstants.Environment.PWD.$$() + "/*"));
  String expectedClasspath = StringUtils.join(ApplicationConstants.CLASS_PATH_SEPARATOR,
      Arrays.asList(ApplicationConstants.Environment.PWD.$$(),
          FRAMEWORK_CLASSPATH, stdClasspath));
  assertEquals("Incorrect classpath with framework and no user precedence",
      expectedClasspath, env.get("CLASSPATH"));

  env.clear();
  conf.setBoolean(MRJobConfig.MAPREDUCE_JOB_USER_CLASSPATH_FIRST, true);
  MRApps.setClasspath(env, conf);
  expectedClasspath = StringUtils.join(ApplicationConstants.CLASS_PATH_SEPARATOR,
      Arrays.asList(ApplicationConstants.Environment.PWD.$$(),
          stdClasspath, FRAMEWORK_CLASSPATH));
  assertEquals("Incorrect classpath with framework and user precedence",
      expectedClasspath, env.get("CLASSPATH"));
}
 

@Test
public void testConcatSourcesParam() {
  final String[] strings = {"/", "/foo", "/bar"};
  for(int n = 0; n < strings.length; n++) {
    final String[] sub = new String[n]; 
    final Path[] paths = new Path[n];
    for(int i = 0; i < paths.length; i++) {
      paths[i] = new Path(sub[i] = strings[i]);
    }

    final String expected = StringUtils.join(",", Arrays.asList(sub));
    final ConcatSourcesParam computed = new ConcatSourcesParam(paths);
    Assert.assertEquals(expected, computed.getValue());
  }
}
 

public void setAccessibleNodeLabels(String queue, Set<String> labels) {
  if (labels == null) {
    return;
  }
  String str = StringUtils.join(",", labels);
  set(getQueuePrefix(queue) + ACCESSIBLE_NODE_LABELS, str);
}
 

public void launchHCatCli(String cmdLine)
  throws IOException {
  String tmpFileName = null;


  String tmpDir = System.getProperty("java.io.tmpdir");
  if (options != null) {
    tmpDir = options.getTempDir();
  }
  tmpFileName =
    new File(tmpDir, "hcat-script-"
      + System.currentTimeMillis()).getAbsolutePath();

  writeHCatScriptFile(tmpFileName, cmdLine);
  // Create the argv for the HCatalog Cli Driver.
  String[] argArray = new String[2];
  argArray[0] = "-f";
  argArray[1] = tmpFileName;
  String argLine = StringUtils.join(",", Arrays.asList(argArray));

  if (testMode) {
    LOG.debug("Executing HCatalog CLI in-process with " + argLine);
    executeHCatProgramInProcess(argArray);
  } else {
    LOG.info("Executing external HCatalog CLI process with args :" + argLine);
    executeExternalHCatProgram(Executor.getCurEnvpStrings(), argArray);
  }
}
 

@Override
public String prettyPrintJournal() {
  return StringUtils.join("\n\t", getStatusJournal());
}
 

private String getUserFileCachePath(String user) {
  return StringUtils.join(Path.SEPARATOR, Arrays.asList(".",
    ContainerLocalizer.USERCACHE, user, ContainerLocalizer.FILECACHE));

}
 

private String getColumnFamiliesAsString() {
  return StringUtils.join(",", getColumnFamilies());
}
 

private String getAppFileCachePath(String user, String appId) {
  return StringUtils.join(Path.SEPARATOR, Arrays.asList(".",
      ContainerLocalizer.USERCACHE, user, ContainerLocalizer.APPCACHE, appId,
      ContainerLocalizer.FILECACHE));
}