Java Code Examples for org.apache.hadoop.conf.Configuration#getStringCollection()

@Test
public void testCsvBlurDriverTest2() throws Exception {
  Configuration configurationSetup = new Configuration();
  ControllerPool controllerPool = new CsvBlurDriver.ControllerPool() {
    @Override
    public Iface getClient(String controllerConnectionStr) {
      return getMockIface();
    }
  };
  AtomicReference<Callable<Void>> ref = new AtomicReference<Callable<Void>>();
  Job job = CsvBlurDriver.setupJob(configurationSetup, controllerPool, ref, "-c", "host:40010", "-d", "family1",
      "col1", "col2", "-d", "family2", "col3", "col4", "-t", "table1", "-i", _path1.toString(), "-i",
      _path2.toString(), "-S", "-C", "1000000", "2000000");
  assertNotNull(job);
  Configuration configuration = job.getConfiguration();
  TableDescriptor tableDescriptor = BlurOutputFormat.getTableDescriptor(configuration);
  assertEquals(tableDescriptor.getName(), "table1");
  Collection<String> inputs = configuration.getStringCollection("mapred.input.dir");
  assertEquals(2, inputs.size());
  Map<String, List<String>> familyAndColumnNameMap = CsvBlurMapper.getFamilyAndColumnNameMap(configuration);
  assertEquals(2, familyAndColumnNameMap.size());
}
 

@Test
public void testCsvBlurDriverTest() throws Exception {
  Configuration configurationSetup = new Configuration();
  ControllerPool controllerPool = new CsvBlurDriver.ControllerPool() {
    @Override
    public Iface getClient(String controllerConnectionStr) {
      return getMockIface();
    }
  };
  AtomicReference<Callable<Void>> ref = new AtomicReference<Callable<Void>>();
  Job job = CsvBlurDriver.setupJob(configurationSetup, controllerPool, ref, "-c", "host:40010", "-d", "family1",
      "col1", "col2", "-d", "family2", "col3", "col4", "-t", "table1", "-i", _path1.toString(), "-i",
      _path2.toString());
  assertNotNull(job);
  Configuration configuration = job.getConfiguration();
  TableDescriptor tableDescriptor = BlurOutputFormat.getTableDescriptor(configuration);
  assertEquals(tableDescriptor.getName(), "table1");
  Collection<String> inputs = configuration.getStringCollection("mapred.input.dir");
  assertEquals(2, inputs.size());
  Map<String, List<String>> familyAndColumnNameMap = CsvBlurMapper.getFamilyAndColumnNameMap(configuration);
  assertEquals(2, familyAndColumnNameMap.size());
}
 

public static void main(String[] args) throws IOException {
  Path p = new Path("./src/main/scripts/conf/hdfs");

  Configuration configuration = mergeHdfsConfigs(p.getFileSystem(new Configuration()), p);

  // configuration.writeXml(System.out);

  Collection<String> nameServices = configuration.getStringCollection(DFS_NAMESERVICES);
  for (String name : nameServices) {
    Path path = new Path("hdfs://" + name + "/");
    FileSystem fileSystem = path.getFileSystem(configuration);
    FileStatus[] listStatus = fileSystem.listStatus(path);
    for (FileStatus fileStatus : listStatus) {
      System.out.println(fileStatus.getPath());
    }
  }
}
 

public static List<KeyProvider> getProviders(Configuration conf
                                             ) throws IOException {
  List<KeyProvider> result = new ArrayList<KeyProvider>();
  for(String path: conf.getStringCollection(KEY_PROVIDER_PATH)) {
    try {
      URI uri = new URI(path);
      KeyProvider kp = get(uri, conf);
      if (kp != null) {
        result.add(kp);
      } else {
        throw new IOException("No KeyProviderFactory for " + uri + " in " +
            KEY_PROVIDER_PATH);
      }
    } catch (URISyntaxException error) {
      throw new IOException("Bad configuration of " + KEY_PROVIDER_PATH +
          " at " + path, error);
    }
  }
  return result;
}
 

public NodeWriter(Configuration conf, String host) {
    super(conf, host);
    String opTypeStr = conf.get(NODE_OPERATION_TYPE);
    if (opTypeStr == null || opTypeStr.isEmpty()) {
        throw new IllegalArgumentException(
                NODE_OPERATION_TYPE + " is not specified.");
    }
    NodeOpType opType = NodeOpType.valueOf(opTypeStr);
    Collection<String> nsCol = conf.getStringCollection(OUTPUT_NAMESPACE);
    StringBuilder buf = new StringBuilder();
    if (nsCol != null) {
        for (Iterator<String> nsIt = nsCol.iterator(); nsIt.hasNext();) {
            String ns = nsIt.next();
            buf.append('"').append(ns).append('"');
            if (nsIt.hasNext()) {
                buf.append(',');
            }
        }
    }
    query = opType.getQuery(buf.toString());
}
 

public static List<CredentialProvider> getProviders(Configuration conf
                                             ) throws IOException {
  List<CredentialProvider> result = new ArrayList<CredentialProvider>();
  for(String path: conf.getStringCollection(CREDENTIAL_PROVIDER_PATH)) {
    try {
      URI uri = new URI(path);
      boolean found = false;
      for(CredentialProviderFactory factory: serviceLoader) {
        CredentialProvider kp = factory.createProvider(uri, conf);
        if (kp != null) {
          result.add(kp);
          found = true;
          break;
        }
      }
      if (!found) {
        throw new IOException("No CredentialProviderFactory for " + uri + " in " +
            CREDENTIAL_PROVIDER_PATH);
      }
    } catch (URISyntaxException error) {
      throw new IOException("Bad configuration of " + CREDENTIAL_PROVIDER_PATH +
          " at " + path, error);
    }
  }
  return result;
}
 

/**
 * Returns a classpath string built from the content of the "tmpjars" value in {@code conf}.
 * Also exposed to shell scripts via `bin/hbase mapredcp`.
 */
public static String buildDependencyClasspath(Configuration conf) {
  if (conf == null) {
    throw new IllegalArgumentException("Must provide a configuration object.");
  }
  Set<String> paths = new HashSet<>(conf.getStringCollection("tmpjars"));
  if (paths.isEmpty()) {
    throw new IllegalArgumentException("Configuration contains no tmpjars.");
  }
  StringBuilder sb = new StringBuilder();
  for (String s : paths) {
    // entries can take the form 'file:/path/to/file.jar'.
    int idx = s.indexOf(":");
    if (idx != -1) s = s.substring(idx + 1);
    if (sb.length() > 0) sb.append(File.pathSeparator);
    sb.append(s);
  }
  return sb.toString();
}
 

@Override
public void init(FileSystem fs, Path path, Configuration conf, boolean overwritable,
    long blocksize) throws IOException, CommonFSUtils.StreamLacksCapabilityException {
  Collection<String> operations = conf.getStringCollection(ALLOWED_OPERATIONS);
  if (operations.isEmpty() || operations.contains(AllowedOperations.all.name())) {
    doAppends = doSyncs = true;
  } else if (operations.contains(AllowedOperations.none.name())) {
    doAppends = doSyncs = false;
  } else {
    doAppends = operations.contains(AllowedOperations.append.name());
    doSyncs = operations.contains(AllowedOperations.sync.name());
  }
  LOG.info("IOTestWriter initialized with appends " + (doAppends ? "enabled" : "disabled") +
      " and syncs " + (doSyncs ? "enabled" : "disabled"));
  super.init(fs, path, conf, overwritable, blocksize);
}
 

@Test
public void testCsvBlurDriverTest3() throws Exception {
  Configuration configurationSetup = new Configuration();
  ControllerPool controllerPool = new CsvBlurDriver.ControllerPool() {
    @Override
    public Iface getClient(String controllerConnectionStr) {
      return getMockIface();
    }
  };
  AtomicReference<Callable<Void>> ref = new AtomicReference<Callable<Void>>();
  Job job = CsvBlurDriver.setupJob(configurationSetup, controllerPool, ref, "-c", "host:40010", "-d", "family1",
      "col1", "col2", "-d", "family2", "col3", "col4", "-t", "table1", "-i", _path1.toString(), "-i",
      _path2.toString(), "-S", "-C", "1000000", "2000000", "-p", "SNAPPY");
  assertNotNull(job);
  Configuration configuration = job.getConfiguration();
  TableDescriptor tableDescriptor = BlurOutputFormat.getTableDescriptor(configuration);
  assertEquals(tableDescriptor.getName(), "table1");
  Collection<String> inputs = configuration.getStringCollection("mapred.input.dir");
  assertEquals(2, inputs.size());
  Map<String, List<String>> familyAndColumnNameMap = CsvBlurMapper.getFamilyAndColumnNameMap(configuration);
  assertEquals(2, familyAndColumnNameMap.size());
  assertEquals("true", configuration.get(CsvBlurDriver.MAPRED_COMPRESS_MAP_OUTPUT));
  assertEquals(SnappyCodec.class.getName(), configuration.get(CsvBlurDriver.MAPRED_MAP_OUTPUT_COMPRESSION_CODEC));
}
 

@Override
public List<InputSplit> getSplits(JobContext context) throws IOException, InterruptedException {
    ArrayList<InputSplit> splits = new ArrayList<>();
    Configuration conf = context.getConfiguration();
    for (String qName : conf.getStringCollection(QUERIES)) {
        int repeatCount = conf.getInt(PREFIX + qName + REPEAT_SUFFIX, 1);
        String query = conf.get(PREFIX + qName + QUERY_SUFFIX);
        for (int i=0; i<repeatCount; i++) {
            splits.add(new QueryInputSplit(qName, query , i));
        }
    }
    return splits;
}
 

public static HConnection createConnection(Configuration conf)
    throws IOException {

  Logger LOG = Logger.getLogger(HConnectionManagerMultiClusterWrapper.class);

  Collection < String > failoverClusters = conf
          .getStringCollection(ConfigConst.HBASE_FAILOVER_CLUSTERS_CONFIG);

  if (failoverClusters.size() == 0) {
    LOG.info(" -- Getting a signle cluster connection !!");
    return HConnectionManager.createConnection(conf);
  } else {

    Map<String, Configuration> configMap = HBaseMultiClusterConfigUtil
        .splitMultiConfigFile(conf);

    LOG.info(" -- Getting primary Connction");
    HConnection primaryConnection = HConnectionManager
        .createConnection(configMap
            .get(HBaseMultiClusterConfigUtil.PRIMARY_NAME));
    LOG.info(" --- Got primary Connction");

    ArrayList<HConnection> failoverConnections = new ArrayList<HConnection>();

    for (Entry<String, Configuration> entry : configMap.entrySet()) {
      if (!entry.getKey().equals(HBaseMultiClusterConfigUtil.PRIMARY_NAME)) {
        LOG.info(" -- Getting failure Connction");
        failoverConnections.add(HConnectionManager.createConnection(entry
            .getValue()));
        LOG.info(" --- Got failover Connction");
      }
    }
    
    return new HConnectionMultiCluster(conf, primaryConnection,
        failoverConnections.toArray(new HConnection[0]));
  }
}
 

static Collection<File> getCheckpointEditsDirs(Configuration conf,
    String defaultName) {
  Collection<String> dirNames = conf
      .getStringCollection("fs.checkpoint.edits.dir");
  if (dirNames.size() == 0 && defaultName != null) {
    dirNames.add(defaultName);
  }
  Collection<File> dirs = new ArrayList<File>(dirNames.size());
  for (String name : dirNames) {
    dirs.add(new File(name));
  }
  return dirs;
}
 

static Collection<File> getCheckpointEditsDirs(Configuration conf,
                                                String defaultName) {
   Collection<String> dirNames = 
               conf.getStringCollection("fs.checkpoint.edits.dir");
if (dirNames.size() == 0 && defaultName != null) {
  dirNames.add(defaultName);
}
Collection<File> dirs = new ArrayList<File>(dirNames.size());
for(String name : dirNames) {
  dirs.add(new File(name));
}
return dirs;    
 }
 

public static void addQuery(Configuration conf, String name, String query, int repeatCount) {
    Collection<String> qNames = conf.getStringCollection(QUERIES);
    qNames.add(name);
    conf.set(PREFIX + name + QUERY_SUFFIX, query);
    conf.setInt(PREFIX + name + REPEAT_SUFFIX, repeatCount);
    conf.setStrings(QUERIES, qNames.toArray(new String[qNames.size()]));
}
 

@Override
protected void setup(Context context) throws IOException, InterruptedException {
  super.setup(context);
  Configuration configuration = context.getConfiguration();
  _autoGenerateRecordIdAsHashOfData = isAutoGenerateRecordIdAsHashOfData(configuration);
  _autoGenerateRowIdAsHashOfData = isAutoGenerateRowIdAsHashOfData(configuration);
  if (_autoGenerateRecordIdAsHashOfData || _autoGenerateRowIdAsHashOfData) {
    try {
      _digest = MessageDigest.getInstance("MD5");
    } catch (NoSuchAlgorithmException e) {
      throw new IOException(e);
    }
  }
  _columnNameMap = getFamilyAndColumnNameMap(configuration);
  _separator = new String(Base64.decodeBase64(configuration.get(BLUR_CSV_SEPARATOR_BASE64, _separator)), UTF_8);
  _splitter = Splitter.on(_separator);
  Path fileCurrentlyProcessing = getCurrentFile(context);
  Collection<String> families = configuration.getStringCollection(BLUR_CSV_FAMILY_PATH_MAPPINGS_FAMILIES);
  OUTER: for (String family : families) {
    Collection<String> pathStrCollection = configuration
        .getStringCollection(BLUR_CSV_FAMILY_PATH_MAPPINGS_FAMILY_PREFIX + family);
    for (String pathStr : pathStrCollection) {
      Path path = new Path(pathStr);
      FileSystem fileSystem = path.getFileSystem(configuration);
      path = path.makeQualified(fileSystem.getUri(), fileSystem.getWorkingDirectory());
      if (isParent(path, fileCurrentlyProcessing)) {
        _familyFromPath = family;
        _familyNotInFile = true;
        break OUTER;
      }
    }
  }
}
 

/**
 * Gets the columns to be mapped over from this configuration object.
 * 
 * @param conf
 *            the Hadoop configuration object
 * @return a set of columns
 * @see #fetchColumns(Configuration, Collection)
 */
protected static Set<Pair<Text,Text>> getFetchedColumns(Configuration conf) {
    Set<Pair<Text,Text>> columns = new HashSet<>();
    for (String col : conf.getStringCollection(COLUMNS)) {
        int idx = col.indexOf(":");
        Text cf = new Text(idx < 0 ? Base64.decodeBase64(col.getBytes()) : Base64.decodeBase64(col.substring(0, idx).getBytes()));
        Text cq = idx < 0 ? null : new Text(Base64.decodeBase64(col.substring(idx + 1).getBytes()));
        columns.add(new Pair<>(cf, cq));
    }
    return columns;
}
 

static Collection<URI> getStorageDirs(Configuration conf) {
  Collection<String> dirNames =
    conf.getStringCollection("dfs.data.dir");
  return Util.stringCollectionAsURIs(dirNames);
}
 

/**
 * Adds the column layout for the given family.
 * 
 * @param configuration
 *          the configuration to apply the layout.
 * @param family
 *          the family name.
 * @param columns
 *          the column names.
 */
public static void addColumns(Configuration configuration, String family, String... columns) {
  Collection<String> families = new TreeSet<String>(configuration.getStringCollection(BLUR_CSV_FAMILIES));
  families.add(family);
  configuration.setStrings(BLUR_CSV_FAMILIES, families.toArray(new String[] {}));
  configuration.setStrings(BLUR_CSV_FAMILY_COLUMN_PREFIX + family, columns);
}
 

/**
 * @param conf Configuration. Please use getRMHAIds to check.
 * @return RM Ids on success
 */
public static Collection<String> getRMHAIds(Configuration conf) {
  return  conf.getStringCollection(YarnConfiguration.RM_HA_IDS);
}
 

/**
 * Returns collection of nameservice Ids from the configuration.
 * @param conf configuration
 * @return collection of nameservice Ids
 */
public static Collection<String> getNameServiceIds(Configuration conf) {
  return conf.getStringCollection(FSConstants.DFS_FEDERATION_NAMESERVICES);
}