org.apache.hadoop.fs.BufferedFSInputStream Java Examples

@Override
public FSDataInputStream open(Path f, int bufferSize) throws IOException {
  if (LOG.isDebugEnabled()) {
    LOG.debug("Opening file: " + f.toString());
  }

  Path absolutePath = makeAbsolute(f);
  String key = pathToKey(absolutePath);
  FileMetadata meta = store.retrieveMetadata(key);
  if (meta == null) {
    throw new FileNotFoundException(f.toString());
  }
  if (meta.isDir()) {
    throw new FileNotFoundException(f.toString()
        + " is a directory not a file.");
  }

  return new FSDataInputStream(new BufferedFSInputStream(
      new NativeAzureFsInputStream(store.retrieve(key), key, meta.getLength()), bufferSize));
}
 

@Override
public FSDataInputStream open(Path f, int bufferSize) throws IOException {
  if (LOG.isDebugEnabled()) {
    LOG.debug("Opening file: " + f.toString());
  }

  Path absolutePath = makeAbsolute(f);
  String key = pathToKey(absolutePath);
  FileMetadata meta = store.retrieveMetadata(key);
  if (meta == null) {
    throw new FileNotFoundException(f.toString());
  }
  if (meta.isDir()) {
    throw new FileNotFoundException(f.toString()
        + " is a directory not a file.");
  }

  return new FSDataInputStream(new BufferedFSInputStream(
      new NativeAzureFsInputStream(store.retrieve(key), key, meta.getLength()), bufferSize));
}
 

@Override
public FSDataInputStream open(Path path, int bufferSize)
        throws IOException {
    try {
        DistributedLogManager dlm = namespace.openLog(getStreamName(path));
        LogReader reader;
        try {
            reader = dlm.openLogReader(DLSN.InitialDLSN);
        } catch (LogNotFoundException lnfe) {
            throw new FileNotFoundException(path.toString());
        } catch (LogEmptyException lee) {
            throw new FileNotFoundException(path.toString());
        }
        return new FSDataInputStream(
            new BufferedFSInputStream(
                new DLInputStream(dlm, reader, 0L),
                bufferSize));
    } catch (LogNotFoundException e) {
        throw new FileNotFoundException(path.toString());
    }
}
 

public HoodieLogFileReader(FileSystem fs, HoodieLogFile logFile, Schema readerSchema, int bufferSize,
    boolean readBlockLazily, boolean reverseReader) throws IOException {
  FSDataInputStream fsDataInputStream = fs.open(logFile.getPath(), bufferSize);
  if (fsDataInputStream.getWrappedStream() instanceof FSInputStream) {
    this.inputStream = new FSDataInputStream(
        new BufferedFSInputStream((FSInputStream) fsDataInputStream.getWrappedStream(), bufferSize));
  } else {
    // fsDataInputStream.getWrappedStream() maybe a BufferedFSInputStream
    // need to wrap in another BufferedFSInputStream the make bufferSize work?
    this.inputStream = fsDataInputStream;
  }

  this.logFile = logFile;
  this.readerSchema = readerSchema;
  this.readBlockLazily = readBlockLazily;
  this.reverseReader = reverseReader;
  if (this.reverseReader) {
    this.reverseLogFilePosition = this.lastReverseLogFilePosition = fs.getFileStatus(logFile.getPath()).getLen();
  }
  addShutDownHook();
}
 

@Override
public FSDataInputStream open(Path path, int bufferSize)
{
    return new FSDataInputStream(
            new BufferedFSInputStream(
                    new PrestoS3InputStream(s3, getBucketName(uri), path, requesterPaysEnabled, maxAttempts, maxBackoffTime, maxRetryTime),
                    bufferSize));
}
 

@Override
public FSDataInputStream open(Path f, int bufferSize) throws IOException {
  FileStatus fs = getFileStatus(f); // will throw if the file doesn't exist
  if (fs.isDirectory()) {
    throw new FileNotFoundException("'" + f + "' is a directory");
  }
  LOG.info("Opening '" + f + "' for reading");
  Path absolutePath = makeAbsolute(f);
  String key = pathToKey(absolutePath);
  return new FSDataInputStream(new BufferedFSInputStream(
      new NativeS3FsInputStream(store, statistics, store.retrieve(key), key), bufferSize));
}
 

@Override
public FSDataInputStream open(Path path, int bufferSize)
    throws IOException
{
  if (skipCache(path.toString(), getConf())) {
    return fs.open(path, bufferSize);
  }

  Path originalPath = new Path(getOriginalURI(path.toUri()).getScheme(), path.toUri().getAuthority(),
      path.toUri().getPath());

  if (CacheConfig.isDummyModeEnabled(this.getConf())) {
    return new FSDataInputStream(
            new BufferedFSInputStream(
                    new DummyModeCachingInputStream(this, originalPath, this.getConf(), statsMBean,
                            clusterManager.getClusterType(), bookKeeperFactory, fs, bufferSize, statistics),
                    CacheConfig.getBlockSize(getConf())));
  }

  try {
    return new FSDataInputStream(
            new BufferedFSInputStream(
                    new CachingInputStream(originalPath, this.getConf(), statsMBean,
                            clusterManager.getClusterType(), bookKeeperFactory, fs, bufferSize, statistics),
                    CacheConfig.getBlockSize(getConf())));
  }
  catch (Exception e) {
    if (CacheConfig.isStrictMode(this.getConf())) {
        log.error("Error in opening Caching Input Stream", e);
        throw e;
    }
    log.warn("Error in opening Caching Input Stream, skipping cache", e);
    return fs.open(path, bufferSize);
  }
}
 

@Override
public FSDataInputStream open(Path f, int bufferSize) throws IOException {
  FileStatus fs = getFileStatus(f); // will throw if the file doesn't exist
  if (fs.isDirectory()) {
    throw new FileNotFoundException("'" + f + "' is a directory");
  }
  LOG.info("Opening '" + f + "' for reading");
  Path absolutePath = makeAbsolute(f);
  String key = pathToKey(absolutePath);
  return new FSDataInputStream(new BufferedFSInputStream(
      new NativeS3FsInputStream(store, statistics, store.retrieve(key), key), bufferSize));
}
 

@Override
public FSDataInputStream open(Path path, int bufferSize) throws IOException {
  SftpGetMonitor monitor = new SftpGetMonitor();
  try {
    ChannelSftp channelSftp = this.fsHelper.getSftpChannel();
    InputStream is = channelSftp.get(HadoopUtils.toUriPath(path), monitor);
    return new FSDataInputStream(new BufferedFSInputStream(new SftpFsHelper.SftpFsFileInputStream(is, channelSftp), bufferSize));
  } catch (SftpException e) {
    throw new IOException(e);
  }
}
 

@Override
public FSDataInputStream open(Path f, int bufferSize) throws IOException {
  if (!exists(f)) {
    throw new FileNotFoundException(f.toString());
  }
  Path absolutePath = makeAbsolute(f);
  String key = pathToKey(absolutePath);
  return new FSDataInputStream(new BufferedFSInputStream(
      new NativeS3FsInputStream(store.retrieve(key), key), bufferSize));
}
 

@Override
public FSDataInputStream open(Path f, int bufferSize) throws IOException {
  if (!exists(f)) {
    throw new FileNotFoundException(f.toString());
  }
  Path absolutePath = makeAbsolute(f);
  String key = pathToKey(absolutePath);
  return new FSDataInputStream(new BufferedFSInputStream(
      new NativeS3FsInputStream(store.retrieve(key), key), bufferSize));
}
 

@Override
public FSDataInputStream open(Path path, int bufferSize) throws IOException {
	if (!exists(path)) {
		throw new FileNotFoundException(path.toString());
	}

	return new FSDataInputStream(new BufferedFSInputStream(
			new LFSInputStream(path), bufferSize));
}