org.apache.hadoop.io.ReadaheadPool Java Examples

The following examples show how to use org.apache.hadoop.io.ReadaheadPool. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: FadvisedFileRegion.java    From hadoop with Apache License 2.0 6 votes vote down vote up 
public FadvisedFileRegion(RandomAccessFile file, long position, long count,
    boolean manageOsCache, int readaheadLength, ReadaheadPool readaheadPool,
    String identifier, int shuffleBufferSize, 
    boolean shuffleTransferToAllowed) throws IOException {
  super(file.getChannel(), position, count);
  this.manageOsCache = manageOsCache;
  this.readaheadLength = readaheadLength;
  this.readaheadPool = readaheadPool;
  this.fd = file.getFD();
  this.identifier = identifier;
  this.fileChannel = file.getChannel();
  this.count = count;
  this.position = position;
  this.shuffleBufferSize = shuffleBufferSize;
  this.shuffleTransferToAllowed = shuffleTransferToAllowed;
}
Example #2
Source File: FadvisedFileRegion.java    From big-c with Apache License 2.0 6 votes vote down vote up 
public FadvisedFileRegion(RandomAccessFile file, long position, long count,
    boolean manageOsCache, int readaheadLength, ReadaheadPool readaheadPool,
    String identifier, int shuffleBufferSize, 
    boolean shuffleTransferToAllowed) throws IOException {
  super(file.getChannel(), position, count);
  this.manageOsCache = manageOsCache;
  this.readaheadLength = readaheadLength;
  this.readaheadPool = readaheadPool;
  this.fd = file.getFD();
  this.identifier = identifier;
  this.fileChannel = file.getChannel();
  this.count = count;
  this.position = position;
  this.shuffleBufferSize = shuffleBufferSize;
  this.shuffleTransferToAllowed = shuffleTransferToAllowed;
}
Example #3
Source File: FadvisedFileRegion.java    From tajo with Apache License 2.0 6 votes vote down vote up 
public FadvisedFileRegion(RandomAccessFile file, long position, long count,
                          boolean manageOsCache, int readaheadLength, ReadaheadPool readaheadPool,
                          String identifier, int shuffleBufferSize,
                          boolean shuffleTransferToAllowed) throws IOException {
  super(file.getChannel(), position, count);
  this.manageOsCache = manageOsCache;
  this.readaheadLength = readaheadLength;
  this.readaheadPool = readaheadPool;
  this.fd = file.getFD();
  this.identifier = identifier;
  this.fileChannel = file.getChannel();
  this.count = count;
  this.position = position;
  this.shuffleBufferSize = shuffleBufferSize;
  this.shuffleTransferToAllowed = shuffleTransferToAllowed;
}
Example #4
Source File: FadvisedFileRegion.java    From tajo with Apache License 2.0 6 votes vote down vote up 
public FadvisedFileRegion(RandomAccessFile file, long position, long count,
                          boolean manageOsCache, int readaheadLength, ReadaheadPool readaheadPool,
                          String identifier, int shuffleBufferSize,
                          boolean shuffleTransferToAllowed) throws IOException {
  super(file.getChannel(), position, count);
  this.manageOsCache = manageOsCache;
  this.readaheadLength = readaheadLength;
  this.readaheadPool = readaheadPool;
  this.fd = file.getFD();
  this.identifier = identifier;
  this.fileChannel = file.getChannel();
  this.count = count;
  this.position = position;
  this.shuffleBufferSize = shuffleBufferSize;
  this.shuffleTransferToAllowed = shuffleTransferToAllowed;
}
Example #5
Source File: FadvisedFileRegion.java    From tez with Apache License 2.0 6 votes vote down vote up 
public FadvisedFileRegion(RandomAccessFile file, long position, long count,
                          boolean manageOsCache, int readaheadLength, ReadaheadPool readaheadPool,
                          String identifier, int shuffleBufferSize,
                          boolean shuffleTransferToAllowed) throws IOException {
  super(file.getChannel(), position, count);
  this.manageOsCache = manageOsCache;
  this.readaheadLength = readaheadLength;
  this.readaheadPool = readaheadPool;
  this.fd = file.getFD();
  this.identifier = identifier;
  this.fileChannel = file.getChannel();
  this.count = count;
  this.position = position;
  this.shuffleBufferSize = shuffleBufferSize;
  this.shuffleTransferToAllowed = shuffleTransferToAllowed;
}
Example #6
Source File: FadvisedChunkedFile.java    From hadoop with Apache License 2.0 5 votes vote down vote up 
public FadvisedChunkedFile(RandomAccessFile file, long position, long count,
    int chunkSize, boolean manageOsCache, int readaheadLength,
    ReadaheadPool readaheadPool, String identifier) throws IOException {
  super(file, position, count, chunkSize);
  this.manageOsCache = manageOsCache;
  this.readaheadLength = readaheadLength;
  this.readaheadPool = readaheadPool;
  this.fd = file.getFD();
  this.identifier = identifier;
}
Example #7
Source File: FadvisedChunkedFile.java    From big-c with Apache License 2.0 5 votes vote down vote up 
public FadvisedChunkedFile(RandomAccessFile file, long position, long count,
    int chunkSize, boolean manageOsCache, int readaheadLength,
    ReadaheadPool readaheadPool, String identifier) throws IOException {
  super(file, position, count, chunkSize);
  this.manageOsCache = manageOsCache;
  this.readaheadLength = readaheadLength;
  this.readaheadPool = readaheadPool;
  this.fd = file.getFD();
  this.identifier = identifier;
}
Example #8
Source File: FadvisedChunkedFile.java    From tajo with Apache License 2.0 5 votes vote down vote up 
public FadvisedChunkedFile(RandomAccessFile file, long position, long count,
                           int chunkSize, boolean manageOsCache, int readaheadLength,
                           ReadaheadPool readaheadPool, String identifier) throws IOException {
  super(file, position, count, chunkSize);
  this.manageOsCache = manageOsCache;
  this.readaheadLength = readaheadLength;
  this.readaheadPool = readaheadPool;
  this.fd = file.getFD();
  this.identifier = identifier;
}
Example #9
Source File: FadvisedChunkedFile.java    From tajo with Apache License 2.0 5 votes vote down vote up 
public FadvisedChunkedFile(RandomAccessFile file, long position, long count,
                           int chunkSize, boolean manageOsCache, int readaheadLength,
                           ReadaheadPool readaheadPool, String identifier) throws IOException {
  super(file, position, count, chunkSize);
  this.manageOsCache = manageOsCache;
  this.readaheadLength = readaheadLength;
  this.readaheadPool = readaheadPool;
  this.fd = file.getFD();
  this.identifier = identifier;
}
Example #10
Source File: FadvisedFileRegion.java    From incubator-tajo with Apache License 2.0 5 votes vote down vote up 
public FadvisedFileRegion(RandomAccessFile file, long position, long count,
    boolean manageOsCache, int readaheadLength, ReadaheadPool readaheadPool,
    String identifier) throws IOException {
  super(file.getChannel(), position, count);
  this.manageOsCache = manageOsCache;
  this.readaheadLength = readaheadLength;
  this.readaheadPool = readaheadPool;
  this.fd = file.getFD();
  this.identifier = identifier;
}
Example #11
Source File: FadvisedChunkedFile.java    From incubator-tajo with Apache License 2.0 5 votes vote down vote up 
public FadvisedChunkedFile(RandomAccessFile file, long position, long count,
    int chunkSize, boolean manageOsCache, int readaheadLength,
    ReadaheadPool readaheadPool, String identifier) throws IOException {
  super(file, position, count, chunkSize);
  this.manageOsCache = manageOsCache;
  this.readaheadLength = readaheadLength;
  this.readaheadPool = readaheadPool;
  this.fd = file.getFD();
  this.identifier = identifier;
}
Example #12
Source File: FadvisedChunkedFile.java    From tez with Apache License 2.0 5 votes vote down vote up 
public FadvisedChunkedFile(RandomAccessFile file, long position, long count,
                           int chunkSize, boolean manageOsCache, int readaheadLength,
                           ReadaheadPool readaheadPool, String identifier) throws IOException {
  super(file, position, count, chunkSize);
  this.manageOsCache = manageOsCache;
  this.readaheadLength = readaheadLength;
  this.readaheadPool = readaheadPool;
  this.fd = file.getFD();
  this.identifier = identifier;
}
Example #13
Source File: DataNode.java    From hadoop with Apache License 2.0 4 votes vote down vote up 
/**
 * This method starts the data node with the specified conf.
 * 
 * @param conf - the configuration
 *  if conf's CONFIG_PROPERTY_SIMULATED property is set
 *  then a simulated storage based data node is created.
 * 
 * @param dataDirs - only for a non-simulated storage data node
 * @throws IOException
 */
void startDataNode(Configuration conf, 
                   List<StorageLocation> dataDirs,
                   SecureResources resources
                   ) throws IOException {

  // settings global for all BPs in the Data Node
  this.secureResources = resources;
  synchronized (this) {
    this.dataDirs = dataDirs;
  }
  this.conf = conf;
  this.dnConf = new DNConf(conf);
  checkSecureConfig(dnConf, conf, resources);

  this.spanReceiverHost =
    SpanReceiverHost.get(conf, DFSConfigKeys.DFS_SERVER_HTRACE_PREFIX);

  if (dnConf.maxLockedMemory > 0) {
    if (!NativeIO.POSIX.getCacheManipulator().verifyCanMlock()) {
      throw new RuntimeException(String.format(
          "Cannot start datanode because the configured max locked memory" +
          " size (%s) is greater than zero and native code is not available.",
          DFS_DATANODE_MAX_LOCKED_MEMORY_KEY));
    }
    if (Path.WINDOWS) {
      NativeIO.Windows.extendWorkingSetSize(dnConf.maxLockedMemory);
    } else {
      long ulimit = NativeIO.POSIX.getCacheManipulator().getMemlockLimit();
      if (dnConf.maxLockedMemory > ulimit) {
        throw new RuntimeException(String.format(
          "Cannot start datanode because the configured max locked memory" +
          " size (%s) of %d bytes is more than the datanode's available" +
          " RLIMIT_MEMLOCK ulimit of %d bytes.",
          DFS_DATANODE_MAX_LOCKED_MEMORY_KEY,
          dnConf.maxLockedMemory,
          ulimit));
      }
    }
  }
  LOG.info("Starting DataNode with maxLockedMemory = " +
      dnConf.maxLockedMemory);

  storage = new DataStorage();
  
  // global DN settings
  registerMXBean();
  initDataXceiver(conf);
  startInfoServer(conf);
  pauseMonitor = new JvmPauseMonitor(conf);
  pauseMonitor.start();

  // BlockPoolTokenSecretManager is required to create ipc server.
  this.blockPoolTokenSecretManager = new BlockPoolTokenSecretManager();

  // Login is done by now. Set the DN user name.
  dnUserName = UserGroupInformation.getCurrentUser().getShortUserName();
  LOG.info("dnUserName = " + dnUserName);
  LOG.info("supergroup = " + supergroup);
  initIpcServer(conf);

  metrics = DataNodeMetrics.create(conf, getDisplayName());
  metrics.getJvmMetrics().setPauseMonitor(pauseMonitor);
  
  blockPoolManager = new BlockPoolManager(this);
  blockPoolManager.refreshNamenodes(conf);

  // Create the ReadaheadPool from the DataNode context so we can
  // exit without having to explicitly shutdown its thread pool.
  readaheadPool = ReadaheadPool.getInstance();
  saslClient = new SaslDataTransferClient(dnConf.conf, 
      dnConf.saslPropsResolver, dnConf.trustedChannelResolver);
  saslServer = new SaslDataTransferServer(dnConf, blockPoolTokenSecretManager);
}
Example #14
Source File: DataNode.java    From big-c with Apache License 2.0 4 votes vote down vote up 
/**
 * This method starts the data node with the specified conf.
 * 
 * @param conf - the configuration
 *  if conf's CONFIG_PROPERTY_SIMULATED property is set
 *  then a simulated storage based data node is created.
 * 
 * @param dataDirs - only for a non-simulated storage data node
 * @throws IOException
 */
void startDataNode(Configuration conf, 
                   List<StorageLocation> dataDirs,
                   SecureResources resources
                   ) throws IOException {

  // settings global for all BPs in the Data Node
  this.secureResources = resources;
  synchronized (this) {
    this.dataDirs = dataDirs;
  }
  this.conf = conf;
  this.dnConf = new DNConf(conf);
  checkSecureConfig(dnConf, conf, resources);

  this.spanReceiverHost =
    SpanReceiverHost.get(conf, DFSConfigKeys.DFS_SERVER_HTRACE_PREFIX);

  if (dnConf.maxLockedMemory > 0) {
    if (!NativeIO.POSIX.getCacheManipulator().verifyCanMlock()) {
      throw new RuntimeException(String.format(
          "Cannot start datanode because the configured max locked memory" +
          " size (%s) is greater than zero and native code is not available.",
          DFS_DATANODE_MAX_LOCKED_MEMORY_KEY));
    }
    if (Path.WINDOWS) {
      NativeIO.Windows.extendWorkingSetSize(dnConf.maxLockedMemory);
    } else {
      long ulimit = NativeIO.POSIX.getCacheManipulator().getMemlockLimit();
      if (dnConf.maxLockedMemory > ulimit) {
        throw new RuntimeException(String.format(
          "Cannot start datanode because the configured max locked memory" +
          " size (%s) of %d bytes is more than the datanode's available" +
          " RLIMIT_MEMLOCK ulimit of %d bytes.",
          DFS_DATANODE_MAX_LOCKED_MEMORY_KEY,
          dnConf.maxLockedMemory,
          ulimit));
      }
    }
  }
  LOG.info("Starting DataNode with maxLockedMemory = " +
      dnConf.maxLockedMemory);

  storage = new DataStorage();
  
  // global DN settings
  registerMXBean();
  initDataXceiver(conf);
  startInfoServer(conf);
  pauseMonitor = new JvmPauseMonitor(conf);
  pauseMonitor.start();

  // BlockPoolTokenSecretManager is required to create ipc server.
  this.blockPoolTokenSecretManager = new BlockPoolTokenSecretManager();

  // Login is done by now. Set the DN user name.
  dnUserName = UserGroupInformation.getCurrentUser().getShortUserName();
  LOG.info("dnUserName = " + dnUserName);
  LOG.info("supergroup = " + supergroup);
  initIpcServer(conf);

  metrics = DataNodeMetrics.create(conf, getDisplayName());
  metrics.getJvmMetrics().setPauseMonitor(pauseMonitor);
  
  blockPoolManager = new BlockPoolManager(this);
  blockPoolManager.refreshNamenodes(conf);

  // Create the ReadaheadPool from the DataNode context so we can
  // exit without having to explicitly shutdown its thread pool.
  readaheadPool = ReadaheadPool.getInstance();
  saslClient = new SaslDataTransferClient(dnConf.conf, 
      dnConf.saslPropsResolver, dnConf.trustedChannelResolver);
  saslServer = new SaslDataTransferServer(dnConf, blockPoolTokenSecretManager);
}
Example #15
Source File: FadvisedFileRegion.java    From tajo with Apache License 2.0 4 votes vote down vote up 
public FadvisedFileRegion(RandomAccessFile file, long position, long count,
                          boolean manageOsCache, int readaheadLength, ReadaheadPool readaheadPool,
                          String identifier) throws IOException {
  this(file, position, count, manageOsCache, readaheadLength, readaheadPool,
      identifier, DEFAULT_SHUFFLE_BUFFER_SIZE, true);
}
Example #16
Source File: FadvisedFileRegion.java    From tajo with Apache License 2.0 4 votes vote down vote up 
public FadvisedFileRegion(RandomAccessFile file, long position, long count,
                          boolean manageOsCache, int readaheadLength, ReadaheadPool readaheadPool,
                          String identifier) throws IOException {
  this(file, position, count, manageOsCache, readaheadLength, readaheadPool,
      identifier, DEFAULT_SHUFFLE_BUFFER_SIZE, true);
}