Java Code Examples for org.apache.hadoop.hdfs.server.namenode.FSNamesystem#getNamespaceEditsDirs()

The following examples show how to use org.apache.hadoop.hdfs.server.namenode.FSNamesystem#getNamespaceEditsDirs() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: TestFailureOfSharedDir.java    From hadoop with Apache License 2.0 6 votes vote down vote up 
/**
 * Multiple shared edits directories is an invalid configuration.
 */
@Test
public void testMultipleSharedDirsFails() throws Exception {
  Configuration conf = new Configuration();
  URI sharedA = new URI("file:///shared-A");
  URI sharedB = new URI("file:///shared-B");
  URI localA = new URI("file:///local-A");

  conf.set(DFSConfigKeys.DFS_NAMENODE_SHARED_EDITS_DIR_KEY,
      Joiner.on(",").join(sharedA,sharedB));
  conf.set(DFSConfigKeys.DFS_NAMENODE_EDITS_DIR_KEY,
      localA.toString());

  try {
    FSNamesystem.getNamespaceEditsDirs(conf);
    fail("Allowed multiple shared edits directories");
  } catch (IOException ioe) {
    assertEquals("Multiple shared edits directories are not yet supported",
        ioe.getMessage());
  }
}
Example 2
Source File: TestFailureOfSharedDir.java    From hadoop with Apache License 2.0 6 votes vote down vote up 
/**
 * Make sure that the shared edits dirs are listed before non-shared dirs
 * when the configuration is parsed. This ensures that the shared journals
 * are synced before the local ones.
 */
@Test
public void testSharedDirsComeFirstInEditsList() throws Exception {
  Configuration conf = new Configuration();
  URI sharedA = new URI("file:///shared-A");
  URI localA = new URI("file:///local-A");
  URI localB = new URI("file:///local-B");
  URI localC = new URI("file:///local-C");
  
  conf.set(DFSConfigKeys.DFS_NAMENODE_SHARED_EDITS_DIR_KEY,
      sharedA.toString());
  // List them in reverse order, to make sure they show up in
  // the order listed, regardless of lexical sort order.
  conf.set(DFSConfigKeys.DFS_NAMENODE_EDITS_DIR_KEY,
      Joiner.on(",").join(localC, localB, localA));
  List<URI> dirs = FSNamesystem.getNamespaceEditsDirs(conf);
  assertEquals(
      "Shared dirs should come first, then local dirs, in the order " +
      "they were listed in the configuration.",
      Joiner.on(",").join(sharedA, localC, localB, localA),
      Joiner.on(",").join(dirs));
}
Example 3
Source File: TestFailureOfSharedDir.java    From big-c with Apache License 2.0 6 votes vote down vote up 
/**
 * Multiple shared edits directories is an invalid configuration.
 */
@Test
public void testMultipleSharedDirsFails() throws Exception {
  Configuration conf = new Configuration();
  URI sharedA = new URI("file:///shared-A");
  URI sharedB = new URI("file:///shared-B");
  URI localA = new URI("file:///local-A");

  conf.set(DFSConfigKeys.DFS_NAMENODE_SHARED_EDITS_DIR_KEY,
      Joiner.on(",").join(sharedA,sharedB));
  conf.set(DFSConfigKeys.DFS_NAMENODE_EDITS_DIR_KEY,
      localA.toString());

  try {
    FSNamesystem.getNamespaceEditsDirs(conf);
    fail("Allowed multiple shared edits directories");
  } catch (IOException ioe) {
    assertEquals("Multiple shared edits directories are not yet supported",
        ioe.getMessage());
  }
}
Example 4
Source File: TestFailureOfSharedDir.java    From big-c with Apache License 2.0 6 votes vote down vote up 
/**
 * Make sure that the shared edits dirs are listed before non-shared dirs
 * when the configuration is parsed. This ensures that the shared journals
 * are synced before the local ones.
 */
@Test
public void testSharedDirsComeFirstInEditsList() throws Exception {
  Configuration conf = new Configuration();
  URI sharedA = new URI("file:///shared-A");
  URI localA = new URI("file:///local-A");
  URI localB = new URI("file:///local-B");
  URI localC = new URI("file:///local-C");
  
  conf.set(DFSConfigKeys.DFS_NAMENODE_SHARED_EDITS_DIR_KEY,
      sharedA.toString());
  // List them in reverse order, to make sure they show up in
  // the order listed, regardless of lexical sort order.
  conf.set(DFSConfigKeys.DFS_NAMENODE_EDITS_DIR_KEY,
      Joiner.on(",").join(localC, localB, localA));
  List<URI> dirs = FSNamesystem.getNamespaceEditsDirs(conf);
  assertEquals(
      "Shared dirs should come first, then local dirs, in the order " +
      "they were listed in the configuration.",
      Joiner.on(",").join(sharedA, localC, localB, localA),
      Joiner.on(",").join(dirs));
}
Example 5
Source File: BootstrapStandby.java    From hadoop with Apache License 2.0 5 votes vote down vote up 
private void parseConfAndFindOtherNN() throws IOException {
  Configuration conf = getConf();
  nsId = DFSUtil.getNamenodeNameServiceId(conf);

  if (!HAUtil.isHAEnabled(conf, nsId)) {
    throw new HadoopIllegalArgumentException(
        "HA is not enabled for this namenode.");
  }
  nnId = HAUtil.getNameNodeId(conf, nsId);
  NameNode.initializeGenericKeys(conf, nsId, nnId);

  if (!HAUtil.usesSharedEditsDir(conf)) {
    throw new HadoopIllegalArgumentException(
      "Shared edits storage is not enabled for this namenode.");
  }
  
  Configuration otherNode = HAUtil.getConfForOtherNode(conf);
  otherNNId = HAUtil.getNameNodeId(otherNode, nsId);
  otherIpcAddr = NameNode.getServiceAddress(otherNode, true);
  Preconditions.checkArgument(otherIpcAddr.getPort() != 0 &&
      !otherIpcAddr.getAddress().isAnyLocalAddress(),
      "Could not determine valid IPC address for other NameNode (%s)" +
      ", got: %s", otherNNId, otherIpcAddr);

  final String scheme = DFSUtil.getHttpClientScheme(conf);
  otherHttpAddr = DFSUtil.getInfoServerWithDefaultHost(
      otherIpcAddr.getHostName(), otherNode, scheme).toURL();

  dirsToFormat = FSNamesystem.getNamespaceDirs(conf);
  editUrisToFormat = FSNamesystem.getNamespaceEditsDirs(
      conf, false);
  sharedEditsUris = FSNamesystem.getSharedEditsDirs(conf);
}
Example 6
Source File: TestHAConfiguration.java    From hadoop with Apache License 2.0 5 votes vote down vote up 
/**
 * Tests that the namenode edits dirs and shared edits dirs are gotten with
 * duplicates removed
 */
@Test
public void testHAUniqueEditDirs() throws IOException {
  Configuration conf = new Configuration();

  conf.set(DFS_NAMENODE_EDITS_DIR_KEY, "file://edits/dir, "
      + "file://edits/shared/dir"); // overlapping
  conf.set(DFS_NAMENODE_SHARED_EDITS_DIR_KEY, "file://edits/shared/dir");

  // getNamespaceEditsDirs removes duplicates across edits and shared.edits
  Collection<URI> editsDirs = FSNamesystem.getNamespaceEditsDirs(conf);
  assertEquals(2, editsDirs.size());
}
Example 7
Source File: BootstrapStandby.java    From big-c with Apache License 2.0 5 votes vote down vote up 
private void parseConfAndFindOtherNN() throws IOException {
  Configuration conf = getConf();
  nsId = DFSUtil.getNamenodeNameServiceId(conf);

  if (!HAUtil.isHAEnabled(conf, nsId)) {
    throw new HadoopIllegalArgumentException(
        "HA is not enabled for this namenode.");
  }
  nnId = HAUtil.getNameNodeId(conf, nsId);
  NameNode.initializeGenericKeys(conf, nsId, nnId);

  if (!HAUtil.usesSharedEditsDir(conf)) {
    throw new HadoopIllegalArgumentException(
      "Shared edits storage is not enabled for this namenode.");
  }
  
  Configuration otherNode = HAUtil.getConfForOtherNode(conf);
  otherNNId = HAUtil.getNameNodeId(otherNode, nsId);
  otherIpcAddr = NameNode.getServiceAddress(otherNode, true);
  Preconditions.checkArgument(otherIpcAddr.getPort() != 0 &&
      !otherIpcAddr.getAddress().isAnyLocalAddress(),
      "Could not determine valid IPC address for other NameNode (%s)" +
      ", got: %s", otherNNId, otherIpcAddr);

  final String scheme = DFSUtil.getHttpClientScheme(conf);
  otherHttpAddr = DFSUtil.getInfoServerWithDefaultHost(
      otherIpcAddr.getHostName(), otherNode, scheme).toURL();

  dirsToFormat = FSNamesystem.getNamespaceDirs(conf);
  editUrisToFormat = FSNamesystem.getNamespaceEditsDirs(
      conf, false);
  sharedEditsUris = FSNamesystem.getSharedEditsDirs(conf);
}
Example 8
Source File: TestHAConfiguration.java    From big-c with Apache License 2.0 5 votes vote down vote up 
/**
 * Tests that the namenode edits dirs and shared edits dirs are gotten with
 * duplicates removed
 */
@Test
public void testHAUniqueEditDirs() throws IOException {
  Configuration conf = new Configuration();

  conf.set(DFS_NAMENODE_EDITS_DIR_KEY, "file://edits/dir, "
      + "file://edits/shared/dir"); // overlapping
  conf.set(DFS_NAMENODE_SHARED_EDITS_DIR_KEY, "file://edits/shared/dir");

  // getNamespaceEditsDirs removes duplicates across edits and shared.edits
  Collection<URI> editsDirs = FSNamesystem.getNamespaceEditsDirs(conf);
  assertEquals(2, editsDirs.size());
}
Example 9
Source File: MiniDFSCluster.java    From hadoop with Apache License 2.0 4 votes vote down vote up 
/**
 * Get the directories where the namenode stores its edits.
 */
public Collection<URI> getNameEditsDirs(int nnIndex) throws IOException {
  return FSNamesystem.getNamespaceEditsDirs(nameNodes[nnIndex].conf);
}
Example 10
Source File: MiniDFSCluster.java    From big-c with Apache License 2.0 4 votes vote down vote up 
/**
 * Get the directories where the namenode stores its edits.
 */
public Collection<URI> getNameEditsDirs(int nnIndex) throws IOException {
  return FSNamesystem.getNamespaceEditsDirs(nameNodes[nnIndex].conf);
}
Example 11
Source File: MiniDFSCluster.java    From RDFS with Apache License 2.0 4 votes vote down vote up 
public Collection<File> getNameEditsDirs(int nnIndex) {
  return FSNamesystem.getNamespaceEditsDirs(nameNodes[nnIndex].conf);
}
Example 12
Source File: MiniDFSCluster.java    From hadoop-gpu with Apache License 2.0 4 votes vote down vote up 
/**
 * Get the directories where the namenode stores its edits.
 */
public Collection<File> getNameEditsDirs() {
  return FSNamesystem.getNamespaceEditsDirs(conf);
}