Java Code Examples for org.apache.hadoop.hdfs.protocol.HdfsConstants#NAMENODE_LAYOUT_VERSION

The following examples show how to use org.apache.hadoop.hdfs.protocol.HdfsConstants#NAMENODE_LAYOUT_VERSION . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: EditLogFileInputStream.java    From hadoop with Apache License 2.0 6 votes vote down vote up 
/**
 * Read the header of fsedit log
 * @param in fsedit stream
 * @return the edit log version number
 * @throws IOException if error occurs
 */
@VisibleForTesting
static int readLogVersion(DataInputStream in, boolean verifyLayoutVersion)
    throws IOException, LogHeaderCorruptException {
  int logVersion;
  try {
    logVersion = in.readInt();
  } catch (EOFException eofe) {
    throw new LogHeaderCorruptException(
        "Reached EOF when reading log header");
  }
  if (verifyLayoutVersion &&
      (logVersion < HdfsConstants.NAMENODE_LAYOUT_VERSION || // future version
       logVersion > Storage.LAST_UPGRADABLE_LAYOUT_VERSION)) { // unsupported
    throw new LogHeaderCorruptException(
        "Unexpected version of the file system log file: "
        + logVersion + ". Current version = "
        + HdfsConstants.NAMENODE_LAYOUT_VERSION + ".");
  }
  return logVersion;
}
Example 2
Source File: EditLogFileInputStream.java    From big-c with Apache License 2.0 6 votes vote down vote up 
/**
 * Read the header of fsedit log
 * @param in fsedit stream
 * @return the edit log version number
 * @throws IOException if error occurs
 */
@VisibleForTesting
static int readLogVersion(DataInputStream in, boolean verifyLayoutVersion)
    throws IOException, LogHeaderCorruptException {
  int logVersion;
  try {
    logVersion = in.readInt();
  } catch (EOFException eofe) {
    throw new LogHeaderCorruptException(
        "Reached EOF when reading log header");
  }
  if (verifyLayoutVersion &&
      (logVersion < HdfsConstants.NAMENODE_LAYOUT_VERSION || // future version
       logVersion > Storage.LAST_UPGRADABLE_LAYOUT_VERSION)) { // unsupported
    throw new LogHeaderCorruptException(
        "Unexpected version of the file system log file: "
        + logVersion + ". Current version = "
        + HdfsConstants.NAMENODE_LAYOUT_VERSION + ".");
  }
  return logVersion;
}
Example 3
Source File: FSEditLogLoader.java    From hadoop with Apache License 2.0 5 votes vote down vote up 
/**
 * Throw appropriate exception during upgrade from 203, when editlog loading
 * could fail due to opcode conflicts.
 */
private void check203UpgradeFailure(int logVersion, Throwable e)
    throws IOException {
  // 0.20.203 version version has conflicting opcodes with the later releases.
  // The editlog must be emptied by restarting the namenode, before proceeding
  // with the upgrade.
  if (Storage.is203LayoutVersion(logVersion)
      && logVersion != HdfsConstants.NAMENODE_LAYOUT_VERSION) {
    String msg = "During upgrade failed to load the editlog version "
        + logVersion + " from release 0.20.203. Please go back to the old "
        + " release and restart the namenode. This empties the editlog "
        + " and saves the namespace. Resume the upgrade after this step.";
    throw new IOException(msg, e);
  }
}
Example 4
Source File: NNStorage.java    From hadoop with Apache License 2.0 5 votes vote down vote up 
public void format() throws IOException {
  this.layoutVersion = HdfsConstants.NAMENODE_LAYOUT_VERSION;
  for (Iterator<StorageDirectory> it =
                         dirIterator(); it.hasNext();) {
    StorageDirectory sd = it.next();
    format(sd);
  }
}
Example 5
Source File: TestMetadataVersionOutput.java    From big-c with Apache License 2.0 5 votes vote down vote up 
@Test(timeout = 30000)
public void testMetadataVersionOutput() throws IOException {

  initConfig();
  dfsCluster = new MiniDFSCluster.Builder(conf).
      manageNameDfsDirs(false).
      numDataNodes(1).
      checkExitOnShutdown(false).
      build();
  dfsCluster.waitClusterUp();
  dfsCluster.shutdown(false);
  initConfig();
  final PrintStream origOut = System.out;
  final ByteArrayOutputStream baos = new ByteArrayOutputStream();
  final PrintStream stdOut = new PrintStream(baos);
  System.setOut(stdOut);
  try {
    NameNode.createNameNode(new String[] { "-metadataVersion" }, conf);
  } catch (Exception e) {
    assertExceptionContains("ExitException", e);
  }
  /* Check if meta data version is printed correctly. */
  final String verNumStr = HdfsConstants.NAMENODE_LAYOUT_VERSION + "";
  assertTrue(baos.toString("UTF-8").
    contains("HDFS Image Version: " + verNumStr));
  assertTrue(baos.toString("UTF-8").
    contains("Software format version: " + verNumStr));
  System.setOut(origOut);
}
Example 6
Source File: NamespaceInfo.java    From hadoop with Apache License 2.0 5 votes vote down vote up 
public NamespaceInfo(int nsID, String clusterID, String bpID,
    long cT, String buildVersion, String softwareVersion,
    long capabilities) {
  super(HdfsConstants.NAMENODE_LAYOUT_VERSION, nsID, clusterID, cT,
      NodeType.NAME_NODE);
  blockPoolID = bpID;
  this.buildVersion = buildVersion;
  this.softwareVersion = softwareVersion;
  this.capabilities = capabilities;
}
Example 7
Source File: TestMetadataVersionOutput.java    From hadoop with Apache License 2.0 5 votes vote down vote up 
@Test(timeout = 30000)
public void testMetadataVersionOutput() throws IOException {

  initConfig();
  dfsCluster = new MiniDFSCluster.Builder(conf).
      manageNameDfsDirs(false).
      numDataNodes(1).
      checkExitOnShutdown(false).
      build();
  dfsCluster.waitClusterUp();
  dfsCluster.shutdown(false);
  initConfig();
  final PrintStream origOut = System.out;
  final ByteArrayOutputStream baos = new ByteArrayOutputStream();
  final PrintStream stdOut = new PrintStream(baos);
  System.setOut(stdOut);
  try {
    NameNode.createNameNode(new String[] { "-metadataVersion" }, conf);
  } catch (Exception e) {
    assertExceptionContains("ExitException", e);
  }
  /* Check if meta data version is printed correctly. */
  final String verNumStr = HdfsConstants.NAMENODE_LAYOUT_VERSION + "";
  assertTrue(baos.toString("UTF-8").
    contains("HDFS Image Version: " + verNumStr));
  assertTrue(baos.toString("UTF-8").
    contains("Software format version: " + verNumStr));
  System.setOut(origOut);
}
Example 8
Source File: NamespaceInfo.java    From big-c with Apache License 2.0 5 votes vote down vote up 
public NamespaceInfo(int nsID, String clusterID, String bpID,
    long cT, String buildVersion, String softwareVersion,
    long capabilities) {
  super(HdfsConstants.NAMENODE_LAYOUT_VERSION, nsID, clusterID, cT,
      NodeType.NAME_NODE);
  blockPoolID = bpID;
  this.buildVersion = buildVersion;
  this.softwareVersion = softwareVersion;
  this.capabilities = capabilities;
}
Example 9
Source File: FSEditLogLoader.java    From big-c with Apache License 2.0 5 votes vote down vote up 
/**
 * Throw appropriate exception during upgrade from 203, when editlog loading
 * could fail due to opcode conflicts.
 */
private void check203UpgradeFailure(int logVersion, Throwable e)
    throws IOException {
  // 0.20.203 version version has conflicting opcodes with the later releases.
  // The editlog must be emptied by restarting the namenode, before proceeding
  // with the upgrade.
  if (Storage.is203LayoutVersion(logVersion)
      && logVersion != HdfsConstants.NAMENODE_LAYOUT_VERSION) {
    String msg = "During upgrade failed to load the editlog version "
        + logVersion + " from release 0.20.203. Please go back to the old "
        + " release and restart the namenode. This empties the editlog "
        + " and saves the namespace. Resume the upgrade after this step.";
    throw new IOException(msg, e);
  }
}
Example 10
Source File: NNStorage.java    From big-c with Apache License 2.0 5 votes vote down vote up 
public void format() throws IOException {
  this.layoutVersion = HdfsConstants.NAMENODE_LAYOUT_VERSION;
  for (Iterator<StorageDirectory> it =
                         dirIterator(); it.hasNext();) {
    StorageDirectory sd = it.next();
    format(sd);
  }
}
Example 11
Source File: BootstrapStandby.java    From hadoop with Apache License 2.0 4 votes vote down vote up 
private boolean checkLayoutVersion(NamespaceInfo nsInfo) throws IOException {
  return (nsInfo.getLayoutVersion() == HdfsConstants.NAMENODE_LAYOUT_VERSION);
}
Example 12
Source File: TestDFSStartupVersions.java    From big-c with Apache License 2.0 4 votes vote down vote up 
/**
 * This test ensures the appropriate response (successful or failure) from 
 * a Datanode when the system is started with differing version combinations. 
 * <pre>
 * For each 3-tuple in the cross product
 *   ({oldLayoutVersion,currentLayoutVersion,futureLayoutVersion},
 *    {currentNamespaceId,incorrectNamespaceId},
 *    {pastFsscTime,currentFsscTime,futureFsscTime})
 *      1. Startup Namenode with version file containing 
 *         (currentLayoutVersion,currentNamespaceId,currentFsscTime)
 *      2. Attempt to startup Datanode with version file containing 
 *         this iterations version 3-tuple
 * </pre>
 */
@Test (timeout=300000)
public void testVersions() throws Exception {
  UpgradeUtilities.initialize();
  Configuration conf = UpgradeUtilities.initializeStorageStateConf(1, 
                                                    new HdfsConfiguration());
  StorageData[] versions = initializeVersions();
  UpgradeUtilities.createNameNodeStorageDirs(
      conf.getStrings(DFSConfigKeys.DFS_NAMENODE_NAME_DIR_KEY), "current");
  cluster = new MiniDFSCluster.Builder(conf).numDataNodes(0)
                                            .format(false)
                                            .manageDataDfsDirs(false)
                                            .manageNameDfsDirs(false)
                                            .startupOption(StartupOption.REGULAR)
                                            .build();
  StorageData nameNodeVersion = new StorageData(
      HdfsConstants.NAMENODE_LAYOUT_VERSION,
      UpgradeUtilities.getCurrentNamespaceID(cluster),
      UpgradeUtilities.getCurrentClusterID(cluster),
      UpgradeUtilities.getCurrentFsscTime(cluster),
      UpgradeUtilities.getCurrentBlockPoolID(cluster));
  
  log("NameNode version info", NAME_NODE, null, nameNodeVersion);
  String bpid = UpgradeUtilities.getCurrentBlockPoolID(cluster);
  for (int i = 0; i < versions.length; i++) {
    File[] storage = UpgradeUtilities.createDataNodeStorageDirs(
        conf.getStrings(DFSConfigKeys.DFS_DATANODE_DATA_DIR_KEY), "current");
    log("DataNode version info", DATA_NODE, i, versions[i]);
    UpgradeUtilities.createDataNodeVersionFile(storage,
        versions[i].storageInfo, bpid, versions[i].blockPoolId);
    try {
      cluster.startDataNodes(conf, 1, false, StartupOption.REGULAR, null);
    } catch (Exception ignore) {
      // Ignore.  The asserts below will check for problems.
      // ignore.printStackTrace();
    }
    assertTrue(cluster.getNameNode() != null);
    assertEquals(isVersionCompatible(nameNodeVersion, versions[i]),
                 cluster.isDataNodeUp());
    cluster.shutdownDataNodes();
  }
}
Example 13
Source File: StorageInfo.java    From big-c with Apache License 2.0 4 votes vote down vote up 
public int getServiceLayoutVersion() {
  return storageType == NodeType.DATA_NODE ? HdfsConstants.DATANODE_LAYOUT_VERSION
      : HdfsConstants.NAMENODE_LAYOUT_VERSION;
}
Example 14
Source File: NameNodeRpcServer.java    From big-c with Apache License 2.0 4 votes vote down vote up 
/**
 * Verify version.
 * @param version layout version
 * @throws IOException on layout version mismatch
 */
void verifyLayoutVersion(int version) throws IOException {
  if (version != HdfsConstants.NAMENODE_LAYOUT_VERSION)
    throw new IncorrectVersionException(
        HdfsConstants.NAMENODE_LAYOUT_VERSION, version, "data node");
}
Example 15
Source File: BootstrapStandby.java    From big-c with Apache License 2.0 4 votes vote down vote up 
private boolean checkLayoutVersion(NamespaceInfo nsInfo) throws IOException {
  return (nsInfo.getLayoutVersion() == HdfsConstants.NAMENODE_LAYOUT_VERSION);
}
Example 16
Source File: TestDFSStartupVersions.java    From hadoop with Apache License 2.0 4 votes vote down vote up 
/**
 * This test ensures the appropriate response (successful or failure) from 
 * a Datanode when the system is started with differing version combinations. 
 * <pre>
 * For each 3-tuple in the cross product
 *   ({oldLayoutVersion,currentLayoutVersion,futureLayoutVersion},
 *    {currentNamespaceId,incorrectNamespaceId},
 *    {pastFsscTime,currentFsscTime,futureFsscTime})
 *      1. Startup Namenode with version file containing 
 *         (currentLayoutVersion,currentNamespaceId,currentFsscTime)
 *      2. Attempt to startup Datanode with version file containing 
 *         this iterations version 3-tuple
 * </pre>
 */
@Test (timeout=300000)
public void testVersions() throws Exception {
  UpgradeUtilities.initialize();
  Configuration conf = UpgradeUtilities.initializeStorageStateConf(1, 
                                                    new HdfsConfiguration());
  StorageData[] versions = initializeVersions();
  UpgradeUtilities.createNameNodeStorageDirs(
      conf.getStrings(DFSConfigKeys.DFS_NAMENODE_NAME_DIR_KEY), "current");
  cluster = new MiniDFSCluster.Builder(conf).numDataNodes(0)
                                            .format(false)
                                            .manageDataDfsDirs(false)
                                            .manageNameDfsDirs(false)
                                            .startupOption(StartupOption.REGULAR)
                                            .build();
  StorageData nameNodeVersion = new StorageData(
      HdfsConstants.NAMENODE_LAYOUT_VERSION,
      UpgradeUtilities.getCurrentNamespaceID(cluster),
      UpgradeUtilities.getCurrentClusterID(cluster),
      UpgradeUtilities.getCurrentFsscTime(cluster),
      UpgradeUtilities.getCurrentBlockPoolID(cluster));
  
  log("NameNode version info", NAME_NODE, null, nameNodeVersion);
  String bpid = UpgradeUtilities.getCurrentBlockPoolID(cluster);
  for (int i = 0; i < versions.length; i++) {
    File[] storage = UpgradeUtilities.createDataNodeStorageDirs(
        conf.getStrings(DFSConfigKeys.DFS_DATANODE_DATA_DIR_KEY), "current");
    log("DataNode version info", DATA_NODE, i, versions[i]);
    UpgradeUtilities.createDataNodeVersionFile(storage,
        versions[i].storageInfo, bpid, versions[i].blockPoolId);
    try {
      cluster.startDataNodes(conf, 1, false, StartupOption.REGULAR, null);
    } catch (Exception ignore) {
      // Ignore.  The asserts below will check for problems.
      // ignore.printStackTrace();
    }
    assertTrue(cluster.getNameNode() != null);
    assertEquals(isVersionCompatible(nameNodeVersion, versions[i]),
                 cluster.isDataNodeUp());
    cluster.shutdownDataNodes();
  }
}
Example 17
Source File: StorageInfo.java    From hadoop with Apache License 2.0 4 votes vote down vote up 
public int getServiceLayoutVersion() {
  return storageType == NodeType.DATA_NODE ? HdfsConstants.DATANODE_LAYOUT_VERSION
      : HdfsConstants.NAMENODE_LAYOUT_VERSION;
}
Example 18
Source File: NameNodeRpcServer.java    From hadoop with Apache License 2.0 4 votes vote down vote up 
/**
 * Verify version.
 * @param version layout version
 * @throws IOException on layout version mismatch
 */
void verifyLayoutVersion(int version) throws IOException {
  if (version != HdfsConstants.NAMENODE_LAYOUT_VERSION)
    throw new IncorrectVersionException(
        HdfsConstants.NAMENODE_LAYOUT_VERSION, version, "data node");
}
Example 19
Source File: UpgradeUtilities.java    From hadoop with Apache License 2.0 2 votes vote down vote up 
/**
 * Return the layout version inherent in the current version
 * of the Namenode, whether it is running or not.
 */
public static int getCurrentNameNodeLayoutVersion() {
  return HdfsConstants.NAMENODE_LAYOUT_VERSION;
}
Example 20
Source File: UpgradeUtilities.java    From big-c with Apache License 2.0 2 votes vote down vote up 
/**
 * Return the layout version inherent in the current version
 * of the Namenode, whether it is running or not.
 */
public static int getCurrentNameNodeLayoutVersion() {
  return HdfsConstants.NAMENODE_LAYOUT_VERSION;
}