org.apache.hadoop.hdfs.util.ByteArray Java Examples

The following examples show how to use org.apache.hadoop.hdfs.util.ByteArray. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: FSDirectory.java    From RDFS with Apache License 2.0 6 votes vote down vote up 
FSDirectory(FSImage fsImage, FSNamesystem ns, Configuration conf) {
  rootDir = new INodeDirectoryWithQuota(INodeDirectory.ROOT_NAME,
      ns.createFsOwnerPermissions(new FsPermission((short)0755)),
      Integer.MAX_VALUE, Long.MAX_VALUE);
  this.fsImage = fsImage;
  this.fsImage.setFSNamesystem(ns);
  int configuredLimit = conf.getInt(
      "dfs.ls.limit", 1000);
  this.lsLimit = configuredLimit>0 ?
      configuredLimit : 1000;
  int threshold = conf.getInt(
      "dfs.namenode.name.cache.threshold",
      10);
  NameNode.LOG.info("Caching file names occuring more than " + threshold
      + " times ");
  nameCache = new NameCache<ByteArray>(threshold);
  initialize(conf);
}
Example #2
Source File: FSDirectory.java    From hadoop with Apache License 2.0 5 votes vote down vote up 
/**
 * Caches frequently used file names to reuse file name objects and
 * reduce heap size.
 */
void cacheName(INode inode) {
  // Name is cached only for files
  if (!inode.isFile()) {
    return;
  }
  ByteArray name = new ByteArray(inode.getLocalNameBytes());
  name = nameCache.put(name);
  if (name != null) {
    inode.setLocalName(name.getBytes());
  }
}
Example #3
Source File: FSDirectory.java    From big-c with Apache License 2.0 5 votes vote down vote up 
/**
 * Caches frequently used file names to reuse file name objects and
 * reduce heap size.
 */
void cacheName(INode inode) {
  // Name is cached only for files
  if (!inode.isFile()) {
    return;
  }
  ByteArray name = new ByteArray(inode.getLocalNameBytes());
  name = nameCache.put(name);
  if (name != null) {
    inode.setLocalName(name.getBytes());
  }
}
Example #4
Source File: FSDirectory.java    From RDFS with Apache License 2.0 5 votes vote down vote up 
/**
 * Caches frequently used file names to reuse file name objects and
 * reduce heap size.
 */
void cacheName(INode inode) {
  // Name is cached only for files
  if (inode.isDirectory()) {
    return;
  }
  ByteArray name = new ByteArray(inode.getLocalNameBytes());
  name = nameCache.put(name);
  if (name != null) {
    inode.setLocalName(name.getBytes());
  }
}
Example #5
Source File: FSDirectory.java    From hadoop with Apache License 2.0 4 votes vote down vote up 
FSDirectory(FSNamesystem ns, Configuration conf) throws IOException {
  this.dirLock = new ReentrantReadWriteLock(true); // fair
  this.inodeId = new INodeId();
  rootDir = createRoot(ns);
  inodeMap = INodeMap.newInstance(rootDir);
  this.isPermissionEnabled = conf.getBoolean(
    DFSConfigKeys.DFS_PERMISSIONS_ENABLED_KEY,
    DFSConfigKeys.DFS_PERMISSIONS_ENABLED_DEFAULT);
  this.fsOwnerShortUserName =
    UserGroupInformation.getCurrentUser().getShortUserName();
  this.supergroup = conf.get(
    DFSConfigKeys.DFS_PERMISSIONS_SUPERUSERGROUP_KEY,
    DFSConfigKeys.DFS_PERMISSIONS_SUPERUSERGROUP_DEFAULT);
  this.aclsEnabled = conf.getBoolean(
      DFSConfigKeys.DFS_NAMENODE_ACLS_ENABLED_KEY,
      DFSConfigKeys.DFS_NAMENODE_ACLS_ENABLED_DEFAULT);
  LOG.info("ACLs enabled? " + aclsEnabled);
  this.xattrsEnabled = conf.getBoolean(
      DFSConfigKeys.DFS_NAMENODE_XATTRS_ENABLED_KEY,
      DFSConfigKeys.DFS_NAMENODE_XATTRS_ENABLED_DEFAULT);
  LOG.info("XAttrs enabled? " + xattrsEnabled);
  this.xattrMaxSize = conf.getInt(
      DFSConfigKeys.DFS_NAMENODE_MAX_XATTR_SIZE_KEY,
      DFSConfigKeys.DFS_NAMENODE_MAX_XATTR_SIZE_DEFAULT);
  Preconditions.checkArgument(xattrMaxSize >= 0,
                              "Cannot set a negative value for the maximum size of an xattr (%s).",
                              DFSConfigKeys.DFS_NAMENODE_MAX_XATTR_SIZE_KEY);
  final String unlimited = xattrMaxSize == 0 ? " (unlimited)" : "";
  LOG.info("Maximum size of an xattr: " + xattrMaxSize + unlimited);

  this.accessTimePrecision = conf.getLong(
      DFS_NAMENODE_ACCESSTIME_PRECISION_KEY,
      DFS_NAMENODE_ACCESSTIME_PRECISION_DEFAULT);

  this.storagePolicyEnabled =
      conf.getBoolean(DFS_STORAGE_POLICY_ENABLED_KEY,
                      DFS_STORAGE_POLICY_ENABLED_DEFAULT);

  this.quotaByStorageTypeEnabled =
      conf.getBoolean(DFS_QUOTA_BY_STORAGETYPE_ENABLED_KEY,
                      DFS_QUOTA_BY_STORAGETYPE_ENABLED_DEFAULT);

  int configuredLimit = conf.getInt(
      DFSConfigKeys.DFS_LIST_LIMIT, DFSConfigKeys.DFS_LIST_LIMIT_DEFAULT);
  this.lsLimit = configuredLimit>0 ?
      configuredLimit : DFSConfigKeys.DFS_LIST_LIMIT_DEFAULT;
  this.contentCountLimit = conf.getInt(
      DFSConfigKeys.DFS_CONTENT_SUMMARY_LIMIT_KEY,
      DFSConfigKeys.DFS_CONTENT_SUMMARY_LIMIT_DEFAULT);
  
  // filesystem limits
  this.maxComponentLength = conf.getInt(
      DFSConfigKeys.DFS_NAMENODE_MAX_COMPONENT_LENGTH_KEY,
      DFSConfigKeys.DFS_NAMENODE_MAX_COMPONENT_LENGTH_DEFAULT);
  this.maxDirItems = conf.getInt(
      DFSConfigKeys.DFS_NAMENODE_MAX_DIRECTORY_ITEMS_KEY,
      DFSConfigKeys.DFS_NAMENODE_MAX_DIRECTORY_ITEMS_DEFAULT);
  this.inodeXAttrsLimit = conf.getInt(
      DFSConfigKeys.DFS_NAMENODE_MAX_XATTRS_PER_INODE_KEY,
      DFSConfigKeys.DFS_NAMENODE_MAX_XATTRS_PER_INODE_DEFAULT);

  Preconditions.checkArgument(this.inodeXAttrsLimit >= 0,
      "Cannot set a negative limit on the number of xattrs per inode (%s).",
      DFSConfigKeys.DFS_NAMENODE_MAX_XATTRS_PER_INODE_KEY);
  // We need a maximum maximum because by default, PB limits message sizes
  // to 64MB. This means we can only store approximately 6.7 million entries
  // per directory, but let's use 6.4 million for some safety.
  final int MAX_DIR_ITEMS = 64 * 100 * 1000;
  Preconditions.checkArgument(
      maxDirItems > 0 && maxDirItems <= MAX_DIR_ITEMS, "Cannot set "
          + DFSConfigKeys.DFS_NAMENODE_MAX_DIRECTORY_ITEMS_KEY
          + " to a value less than 1 or greater than " + MAX_DIR_ITEMS);

  int threshold = conf.getInt(
      DFSConfigKeys.DFS_NAMENODE_NAME_CACHE_THRESHOLD_KEY,
      DFSConfigKeys.DFS_NAMENODE_NAME_CACHE_THRESHOLD_DEFAULT);
  NameNode.LOG.info("Caching file names occuring more than " + threshold
      + " times");
  nameCache = new NameCache<ByteArray>(threshold);
  namesystem = ns;
  this.editLog = ns.getEditLog();
  ezManager = new EncryptionZoneManager(this, conf);
}
Example #6
Source File: FSDirectory.java    From big-c with Apache License 2.0 4 votes vote down vote up 
FSDirectory(FSNamesystem ns, Configuration conf) throws IOException {
  this.dirLock = new ReentrantReadWriteLock(true); // fair
  this.inodeId = new INodeId();
  rootDir = createRoot(ns);
  inodeMap = INodeMap.newInstance(rootDir);
  this.isPermissionEnabled = conf.getBoolean(
    DFSConfigKeys.DFS_PERMISSIONS_ENABLED_KEY,
    DFSConfigKeys.DFS_PERMISSIONS_ENABLED_DEFAULT);
  this.fsOwnerShortUserName =
    UserGroupInformation.getCurrentUser().getShortUserName();
  this.supergroup = conf.get(
    DFSConfigKeys.DFS_PERMISSIONS_SUPERUSERGROUP_KEY,
    DFSConfigKeys.DFS_PERMISSIONS_SUPERUSERGROUP_DEFAULT);
  this.aclsEnabled = conf.getBoolean(
      DFSConfigKeys.DFS_NAMENODE_ACLS_ENABLED_KEY,
      DFSConfigKeys.DFS_NAMENODE_ACLS_ENABLED_DEFAULT);
  LOG.info("ACLs enabled? " + aclsEnabled);
  this.xattrsEnabled = conf.getBoolean(
      DFSConfigKeys.DFS_NAMENODE_XATTRS_ENABLED_KEY,
      DFSConfigKeys.DFS_NAMENODE_XATTRS_ENABLED_DEFAULT);
  LOG.info("XAttrs enabled? " + xattrsEnabled);
  this.xattrMaxSize = conf.getInt(
      DFSConfigKeys.DFS_NAMENODE_MAX_XATTR_SIZE_KEY,
      DFSConfigKeys.DFS_NAMENODE_MAX_XATTR_SIZE_DEFAULT);
  Preconditions.checkArgument(xattrMaxSize >= 0,
                              "Cannot set a negative value for the maximum size of an xattr (%s).",
                              DFSConfigKeys.DFS_NAMENODE_MAX_XATTR_SIZE_KEY);
  final String unlimited = xattrMaxSize == 0 ? " (unlimited)" : "";
  LOG.info("Maximum size of an xattr: " + xattrMaxSize + unlimited);

  this.accessTimePrecision = conf.getLong(
      DFS_NAMENODE_ACCESSTIME_PRECISION_KEY,
      DFS_NAMENODE_ACCESSTIME_PRECISION_DEFAULT);

  this.storagePolicyEnabled =
      conf.getBoolean(DFS_STORAGE_POLICY_ENABLED_KEY,
                      DFS_STORAGE_POLICY_ENABLED_DEFAULT);

  this.quotaByStorageTypeEnabled =
      conf.getBoolean(DFS_QUOTA_BY_STORAGETYPE_ENABLED_KEY,
                      DFS_QUOTA_BY_STORAGETYPE_ENABLED_DEFAULT);

  int configuredLimit = conf.getInt(
      DFSConfigKeys.DFS_LIST_LIMIT, DFSConfigKeys.DFS_LIST_LIMIT_DEFAULT);
  this.lsLimit = configuredLimit>0 ?
      configuredLimit : DFSConfigKeys.DFS_LIST_LIMIT_DEFAULT;
  this.contentCountLimit = conf.getInt(
      DFSConfigKeys.DFS_CONTENT_SUMMARY_LIMIT_KEY,
      DFSConfigKeys.DFS_CONTENT_SUMMARY_LIMIT_DEFAULT);
  
  // filesystem limits
  this.maxComponentLength = conf.getInt(
      DFSConfigKeys.DFS_NAMENODE_MAX_COMPONENT_LENGTH_KEY,
      DFSConfigKeys.DFS_NAMENODE_MAX_COMPONENT_LENGTH_DEFAULT);
  this.maxDirItems = conf.getInt(
      DFSConfigKeys.DFS_NAMENODE_MAX_DIRECTORY_ITEMS_KEY,
      DFSConfigKeys.DFS_NAMENODE_MAX_DIRECTORY_ITEMS_DEFAULT);
  this.inodeXAttrsLimit = conf.getInt(
      DFSConfigKeys.DFS_NAMENODE_MAX_XATTRS_PER_INODE_KEY,
      DFSConfigKeys.DFS_NAMENODE_MAX_XATTRS_PER_INODE_DEFAULT);

  Preconditions.checkArgument(this.inodeXAttrsLimit >= 0,
      "Cannot set a negative limit on the number of xattrs per inode (%s).",
      DFSConfigKeys.DFS_NAMENODE_MAX_XATTRS_PER_INODE_KEY);
  // We need a maximum maximum because by default, PB limits message sizes
  // to 64MB. This means we can only store approximately 6.7 million entries
  // per directory, but let's use 6.4 million for some safety.
  final int MAX_DIR_ITEMS = 64 * 100 * 1000;
  Preconditions.checkArgument(
      maxDirItems > 0 && maxDirItems <= MAX_DIR_ITEMS, "Cannot set "
          + DFSConfigKeys.DFS_NAMENODE_MAX_DIRECTORY_ITEMS_KEY
          + " to a value less than 1 or greater than " + MAX_DIR_ITEMS);

  int threshold = conf.getInt(
      DFSConfigKeys.DFS_NAMENODE_NAME_CACHE_THRESHOLD_KEY,
      DFSConfigKeys.DFS_NAMENODE_NAME_CACHE_THRESHOLD_DEFAULT);
  NameNode.LOG.info("Caching file names occuring more than " + threshold
      + " times");
  nameCache = new NameCache<ByteArray>(threshold);
  namesystem = ns;
  this.editLog = ns.getEditLog();
  ezManager = new EncryptionZoneManager(this, conf);
}