org.apache.hadoop.util.PerformanceAdvisory Java Examples

The following examples show how to use org.apache.hadoop.util.PerformanceAdvisory. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: DomainSocketFactory.java    From hadoop with Apache License 2.0 6 votes vote down vote up 
public DomainSocketFactory(Conf conf) {
  final String feature;
  if (conf.isShortCircuitLocalReads() && (!conf.isUseLegacyBlockReaderLocal())) {
    feature = "The short-circuit local reads feature";
  } else if (conf.isDomainSocketDataTraffic()) {
    feature = "UNIX domain socket data traffic";
  } else {
    feature = null;
  }

  if (feature == null) {
    PerformanceAdvisory.LOG.debug(
        "Both short-circuit local reads and UNIX domain socket are disabled.");
  } else {
    if (conf.getDomainSocketPath().isEmpty()) {
      throw new HadoopIllegalArgumentException(feature + " is enabled but "
          + DFSConfigKeys.DFS_DOMAIN_SOCKET_PATH_KEY + " is not set.");
    } else if (DomainSocket.getLoadingFailureReason() != null) {
      LOG.warn(feature + " cannot be used because "
          + DomainSocket.getLoadingFailureReason());
    } else {
      LOG.debug(feature + " is enabled.");
    }
  }
}
Example #2
Source File: DomainSocketFactory.java    From big-c with Apache License 2.0 6 votes vote down vote up 
public DomainSocketFactory(Conf conf) {
  final String feature;
  if (conf.isShortCircuitLocalReads() && (!conf.isUseLegacyBlockReaderLocal())) {
    feature = "The short-circuit local reads feature";
  } else if (conf.isDomainSocketDataTraffic()) {
    feature = "UNIX domain socket data traffic";
  } else {
    feature = null;
  }

  if (feature == null) {
    PerformanceAdvisory.LOG.debug(
        "Both short-circuit local reads and UNIX domain socket are disabled.");
  } else {
    if (conf.getDomainSocketPath().isEmpty()) {
      throw new HadoopIllegalArgumentException(feature + " is enabled but "
          + DFSConfigKeys.DFS_DOMAIN_SOCKET_PATH_KEY + " is not set.");
    } else if (DomainSocket.getLoadingFailureReason() != null) {
      LOG.warn(feature + " cannot be used because "
          + DomainSocket.getLoadingFailureReason());
    } else {
      LOG.debug(feature + " is enabled.");
    }
  }
}
Example #3
Source File: JniBasedUnixGroupsMappingWithFallback.java    From hadoop with Apache License 2.0 5 votes vote down vote up 
public JniBasedUnixGroupsMappingWithFallback() {
  if (NativeCodeLoader.isNativeCodeLoaded()) {
    this.impl = new JniBasedUnixGroupsMapping();
  } else {
    PerformanceAdvisory.LOG.debug("Falling back to shell based");
    this.impl = new ShellBasedUnixGroupsMapping();
  }
  if (LOG.isDebugEnabled()){
    LOG.debug("Group mapping impl=" + impl.getClass().getName());
  }
}
Example #4
Source File: CryptoCodec.java    From hadoop with Apache License 2.0 5 votes vote down vote up 
/**
 * Get crypto codec for specified algorithm/mode/padding.
 * 
 * @param conf
 *          the configuration
 * @param cipherSuite
 *          algorithm/mode/padding
 * @return CryptoCodec the codec object. Null value will be returned if no
 *         crypto codec classes with cipher suite configured.
 */
public static CryptoCodec getInstance(Configuration conf, 
    CipherSuite cipherSuite) {
  List<Class<? extends CryptoCodec>> klasses = getCodecClasses(
      conf, cipherSuite);
  if (klasses == null) {
    return null;
  }
  CryptoCodec codec = null;
  for (Class<? extends CryptoCodec> klass : klasses) {
    try {
      CryptoCodec c = ReflectionUtils.newInstance(klass, conf);
      if (c.getCipherSuite().getName().equals(cipherSuite.getName())) {
        if (codec == null) {
          PerformanceAdvisory.LOG.debug("Using crypto codec {}.",
              klass.getName());
          codec = c;
        }
      } else {
        PerformanceAdvisory.LOG.debug(
            "Crypto codec {} doesn't meet the cipher suite {}.",
            klass.getName(), cipherSuite.getName());
      }
    } catch (Exception e) {
      PerformanceAdvisory.LOG.debug("Crypto codec {} is not available.",
          klass.getName());
    }
  }
  
  return codec;
}
Example #5
Source File: JniBasedUnixGroupsMappingWithFallback.java    From big-c with Apache License 2.0 5 votes vote down vote up 
public JniBasedUnixGroupsMappingWithFallback() {
  if (NativeCodeLoader.isNativeCodeLoaded()) {
    this.impl = new JniBasedUnixGroupsMapping();
  } else {
    PerformanceAdvisory.LOG.debug("Falling back to shell based");
    this.impl = new ShellBasedUnixGroupsMapping();
  }
  if (LOG.isDebugEnabled()){
    LOG.debug("Group mapping impl=" + impl.getClass().getName());
  }
}
Example #6
Source File: CryptoCodec.java    From big-c with Apache License 2.0 5 votes vote down vote up 
/**
 * Get crypto codec for specified algorithm/mode/padding.
 * 
 * @param conf
 *          the configuration
 * @param cipherSuite
 *          algorithm/mode/padding
 * @return CryptoCodec the codec object. Null value will be returned if no
 *         crypto codec classes with cipher suite configured.
 */
public static CryptoCodec getInstance(Configuration conf, 
    CipherSuite cipherSuite) {
  List<Class<? extends CryptoCodec>> klasses = getCodecClasses(
      conf, cipherSuite);
  if (klasses == null) {
    return null;
  }
  CryptoCodec codec = null;
  for (Class<? extends CryptoCodec> klass : klasses) {
    try {
      CryptoCodec c = ReflectionUtils.newInstance(klass, conf);
      if (c.getCipherSuite().getName().equals(cipherSuite.getName())) {
        if (codec == null) {
          PerformanceAdvisory.LOG.debug("Using crypto codec {}.",
              klass.getName());
          codec = c;
        }
      } else {
        PerformanceAdvisory.LOG.debug(
            "Crypto codec {} doesn't meet the cipher suite {}.",
            klass.getName(), cipherSuite.getName());
      }
    } catch (Exception e) {
      PerformanceAdvisory.LOG.debug("Crypto codec {} is not available.",
          klass.getName());
    }
  }
  
  return codec;
}
Example #7
Source File: BlockReaderFactory.java    From hadoop with Apache License 2.0 4 votes vote down vote up 
/**
 * Get {@link BlockReaderLocalLegacy} for short circuited local reads.
 * This block reader implements the path-based style of local reads
 * first introduced in HDFS-2246.
 */
private BlockReader getLegacyBlockReaderLocal() throws IOException {
  if (LOG.isTraceEnabled()) {
    LOG.trace(this + ": trying to construct BlockReaderLocalLegacy");
  }
  if (!DFSClient.isLocalAddress(inetSocketAddress)) {
    if (LOG.isTraceEnabled()) {
      LOG.trace(this + ": can't construct BlockReaderLocalLegacy because " +
          "the address " + inetSocketAddress + " is not local");
    }
    return null;
  }
  if (clientContext.getDisableLegacyBlockReaderLocal()) {
    PerformanceAdvisory.LOG.debug(this + ": can't construct " +
        "BlockReaderLocalLegacy because " +
        "disableLegacyBlockReaderLocal is set.");
    return null;
  }
  IOException ioe = null;
  try {
    return BlockReaderLocalLegacy.newBlockReader(conf,
        userGroupInformation, configuration, fileName, block, token,
        datanode, startOffset, length, storageType);
  } catch (RemoteException remoteException) {
    ioe = remoteException.unwrapRemoteException(
              InvalidToken.class, AccessControlException.class);
  } catch (IOException e) {
    ioe = e;
  }
  if ((!(ioe instanceof AccessControlException)) &&
      isSecurityException(ioe)) {
    // Handle security exceptions.
    // We do not handle AccessControlException here, since
    // BlockReaderLocalLegacy#newBlockReader uses that exception to indicate
    // that the user is not in dfs.block.local-path-access.user, a condition
    // which requires us to disable legacy SCR.
    throw ioe;
  }
  LOG.warn(this + ": error creating legacy BlockReaderLocal.  " +
      "Disabling legacy local reads.", ioe);
  clientContext.setDisableLegacyBlockReaderLocal();
  return null;
}
Example #8
Source File: BlockReaderFactory.java    From hadoop with Apache License 2.0 4 votes vote down vote up 
private BlockReader getBlockReaderLocal() throws InvalidToken {
  if (LOG.isTraceEnabled()) {
    LOG.trace(this + ": trying to construct a BlockReaderLocal " +
        "for short-circuit reads.");
  }
  if (pathInfo == null) {
    pathInfo = clientContext.getDomainSocketFactory().
                    getPathInfo(inetSocketAddress, conf);
  }
  if (!pathInfo.getPathState().getUsableForShortCircuit()) {
    PerformanceAdvisory.LOG.debug(this + ": " + pathInfo + " is not " +
        "usable for short circuit; giving up on BlockReaderLocal.");
    return null;
  }
  ShortCircuitCache cache = clientContext.getShortCircuitCache();
  ExtendedBlockId key = new ExtendedBlockId(block.getBlockId(), block.getBlockPoolId());
  ShortCircuitReplicaInfo info = cache.fetchOrCreate(key, this);
  InvalidToken exc = info.getInvalidTokenException();
  if (exc != null) {
    if (LOG.isTraceEnabled()) {
      LOG.trace(this + ": got InvalidToken exception while trying to " +
          "construct BlockReaderLocal via " + pathInfo.getPath());
    }
    throw exc;
  }
  if (info.getReplica() == null) {
    if (LOG.isTraceEnabled()) {
      PerformanceAdvisory.LOG.debug(this + ": failed to get " +
          "ShortCircuitReplica. Cannot construct " +
          "BlockReaderLocal via " + pathInfo.getPath());
    }
    return null;
  }
  return new BlockReaderLocal.Builder(conf).
      setFilename(fileName).
      setBlock(block).
      setStartOffset(startOffset).
      setShortCircuitReplica(info.getReplica()).
      setVerifyChecksum(verifyChecksum).
      setCachingStrategy(cachingStrategy).
      setStorageType(storageType).
      build();
}
Example #9
Source File: BlockReaderFactory.java    From hadoop with Apache License 2.0 4 votes vote down vote up 
/**
 * Get a RemoteBlockReader that communicates over a UNIX domain socket.
 *
 * @return The new BlockReader, or null if we failed to create the block
 * reader.
 *
 * @throws InvalidToken    If the block token was invalid.
 * Potentially other security-related execptions.
 */
private BlockReader getRemoteBlockReaderFromDomain() throws IOException {
  if (pathInfo == null) {
    pathInfo = clientContext.getDomainSocketFactory().
                    getPathInfo(inetSocketAddress, conf);
  }
  if (!pathInfo.getPathState().getUsableForDataTransfer()) {
    PerformanceAdvisory.LOG.debug(this + ": not trying to create a " +
        "remote block reader because the UNIX domain socket at " +
        pathInfo + " is not usable.");
    return null;
  }
  if (LOG.isTraceEnabled()) {
    LOG.trace(this + ": trying to create a remote block reader from the " +
        "UNIX domain socket at " + pathInfo.getPath());
  }

  while (true) {
    BlockReaderPeer curPeer = nextDomainPeer();
    if (curPeer == null) break;
    if (curPeer.fromCache) remainingCacheTries--;
    DomainPeer peer = (DomainPeer)curPeer.peer;
    BlockReader blockReader = null;
    try {
      blockReader = getRemoteBlockReader(peer);
      return blockReader;
    } catch (IOException ioe) {
      IOUtils.cleanup(LOG, peer);
      if (isSecurityException(ioe)) {
        if (LOG.isTraceEnabled()) {
          LOG.trace(this + ": got security exception while constructing " +
              "a remote block reader from the unix domain socket at " +
              pathInfo.getPath(), ioe);
        }
        throw ioe;
      }
      if (curPeer.fromCache) {
        // Handle an I/O error we got when using a cached peer.  These are
        // considered less serious, because the underlying socket may be stale.
        if (LOG.isDebugEnabled()) {
          LOG.debug("Closed potentially stale domain peer " + peer, ioe);
        }
      } else {
        // Handle an I/O error we got when using a newly created domain peer.
        // We temporarily disable the domain socket path for a few minutes in
        // this case, to prevent wasting more time on it.
        LOG.warn("I/O error constructing remote block reader.  Disabling " +
            "domain socket " + peer.getDomainSocket(), ioe);
        clientContext.getDomainSocketFactory()
            .disableDomainSocketPath(pathInfo.getPath());
        return null;
      }
    } finally {
      if (blockReader == null) {
        IOUtils.cleanup(LOG, peer);
      }
    }
  }
  return null;
}
Example #10
Source File: BlockReaderFactory.java    From big-c with Apache License 2.0 4 votes vote down vote up 
/**
 * Get {@link BlockReaderLocalLegacy} for short circuited local reads.
 * This block reader implements the path-based style of local reads
 * first introduced in HDFS-2246.
 */
private BlockReader getLegacyBlockReaderLocal() throws IOException {
  if (LOG.isTraceEnabled()) {
    LOG.trace(this + ": trying to construct BlockReaderLocalLegacy");
  }
  if (!DFSClient.isLocalAddress(inetSocketAddress)) {
    if (LOG.isTraceEnabled()) {
      LOG.trace(this + ": can't construct BlockReaderLocalLegacy because " +
          "the address " + inetSocketAddress + " is not local");
    }
    return null;
  }
  if (clientContext.getDisableLegacyBlockReaderLocal()) {
    PerformanceAdvisory.LOG.debug(this + ": can't construct " +
        "BlockReaderLocalLegacy because " +
        "disableLegacyBlockReaderLocal is set.");
    return null;
  }
  IOException ioe = null;
  try {
    return BlockReaderLocalLegacy.newBlockReader(conf,
        userGroupInformation, configuration, fileName, block, token,
        datanode, startOffset, length, storageType);
  } catch (RemoteException remoteException) {
    ioe = remoteException.unwrapRemoteException(
              InvalidToken.class, AccessControlException.class);
  } catch (IOException e) {
    ioe = e;
  }
  if ((!(ioe instanceof AccessControlException)) &&
      isSecurityException(ioe)) {
    // Handle security exceptions.
    // We do not handle AccessControlException here, since
    // BlockReaderLocalLegacy#newBlockReader uses that exception to indicate
    // that the user is not in dfs.block.local-path-access.user, a condition
    // which requires us to disable legacy SCR.
    throw ioe;
  }
  LOG.warn(this + ": error creating legacy BlockReaderLocal.  " +
      "Disabling legacy local reads.", ioe);
  clientContext.setDisableLegacyBlockReaderLocal();
  return null;
}
Example #11
Source File: BlockReaderFactory.java    From big-c with Apache License 2.0 4 votes vote down vote up 
private BlockReader getBlockReaderLocal() throws InvalidToken {
  if (LOG.isTraceEnabled()) {
    LOG.trace(this + ": trying to construct a BlockReaderLocal " +
        "for short-circuit reads.");
  }
  if (pathInfo == null) {
    pathInfo = clientContext.getDomainSocketFactory().
                    getPathInfo(inetSocketAddress, conf);
  }
  if (!pathInfo.getPathState().getUsableForShortCircuit()) {
    PerformanceAdvisory.LOG.debug(this + ": " + pathInfo + " is not " +
        "usable for short circuit; giving up on BlockReaderLocal.");
    return null;
  }
  ShortCircuitCache cache = clientContext.getShortCircuitCache();
  ExtendedBlockId key = new ExtendedBlockId(block.getBlockId(), block.getBlockPoolId());
  ShortCircuitReplicaInfo info = cache.fetchOrCreate(key, this);
  InvalidToken exc = info.getInvalidTokenException();
  if (exc != null) {
    if (LOG.isTraceEnabled()) {
      LOG.trace(this + ": got InvalidToken exception while trying to " +
          "construct BlockReaderLocal via " + pathInfo.getPath());
    }
    throw exc;
  }
  if (info.getReplica() == null) {
    if (LOG.isTraceEnabled()) {
      PerformanceAdvisory.LOG.debug(this + ": failed to get " +
          "ShortCircuitReplica. Cannot construct " +
          "BlockReaderLocal via " + pathInfo.getPath());
    }
    return null;
  }
  return new BlockReaderLocal.Builder(conf).
      setFilename(fileName).
      setBlock(block).
      setStartOffset(startOffset).
      setShortCircuitReplica(info.getReplica()).
      setVerifyChecksum(verifyChecksum).
      setCachingStrategy(cachingStrategy).
      setStorageType(storageType).
      build();
}
Example #12
Source File: BlockReaderFactory.java    From big-c with Apache License 2.0 4 votes vote down vote up 
/**
 * Get a RemoteBlockReader that communicates over a UNIX domain socket.
 *
 * @return The new BlockReader, or null if we failed to create the block
 * reader.
 *
 * @throws InvalidToken    If the block token was invalid.
 * Potentially other security-related execptions.
 */
private BlockReader getRemoteBlockReaderFromDomain() throws IOException {
  if (pathInfo == null) {
    pathInfo = clientContext.getDomainSocketFactory().
                    getPathInfo(inetSocketAddress, conf);
  }
  if (!pathInfo.getPathState().getUsableForDataTransfer()) {
    PerformanceAdvisory.LOG.debug(this + ": not trying to create a " +
        "remote block reader because the UNIX domain socket at " +
        pathInfo + " is not usable.");
    return null;
  }
  if (LOG.isTraceEnabled()) {
    LOG.trace(this + ": trying to create a remote block reader from the " +
        "UNIX domain socket at " + pathInfo.getPath());
  }

  while (true) {
    BlockReaderPeer curPeer = nextDomainPeer();
    if (curPeer == null) break;
    if (curPeer.fromCache) remainingCacheTries--;
    DomainPeer peer = (DomainPeer)curPeer.peer;
    BlockReader blockReader = null;
    try {
      blockReader = getRemoteBlockReader(peer);
      return blockReader;
    } catch (IOException ioe) {
      IOUtils.cleanup(LOG, peer);
      if (isSecurityException(ioe)) {
        if (LOG.isTraceEnabled()) {
          LOG.trace(this + ": got security exception while constructing " +
              "a remote block reader from the unix domain socket at " +
              pathInfo.getPath(), ioe);
        }
        throw ioe;
      }
      if (curPeer.fromCache) {
        // Handle an I/O error we got when using a cached peer.  These are
        // considered less serious, because the underlying socket may be stale.
        if (LOG.isDebugEnabled()) {
          LOG.debug("Closed potentially stale domain peer " + peer, ioe);
        }
      } else {
        // Handle an I/O error we got when using a newly created domain peer.
        // We temporarily disable the domain socket path for a few minutes in
        // this case, to prevent wasting more time on it.
        LOG.warn("I/O error constructing remote block reader.  Disabling " +
            "domain socket " + peer.getDomainSocket(), ioe);
        clientContext.getDomainSocketFactory()
            .disableDomainSocketPath(pathInfo.getPath());
        return null;
      }
    } finally {
      if (blockReader == null) {
        IOUtils.cleanup(LOG, peer);
      }
    }
  }
  return null;
}