Java Code Examples for org.apache.hadoop.fs.FileSystem#getDefaultReplication()

/**
 * Executes the filesystem operation.
 *
 * @param fs filesystem instance to use.
 *
 * @return The URI of the created file.
 *
 * @throws IOException thrown if an IO error occured.
 */
@Override
public Void execute(FileSystem fs) throws IOException {
  if (replication == -1) {
    replication = fs.getDefaultReplication(path);
  }
  if (blockSize == -1) {
    blockSize = fs.getDefaultBlockSize(path);
  }
  FsPermission fsPermission = new FsPermission(permission);
  int bufferSize = fs.getConf().getInt("httpfs.buffer.size", 4096);
  OutputStream os = fs.create(path, fsPermission, override, bufferSize, replication, blockSize, null);
  IOUtils.copyBytes(is, os, bufferSize, true);
  os.close();
  return null;
}
 

/**
 * Executes the filesystem operation.
 *
 * @param fs filesystem instance to use.
 *
 * @return The URI of the created file.
 *
 * @throws IOException thrown if an IO error occured.
 */
@Override
public Void execute(FileSystem fs) throws IOException {
  if (replication == -1) {
    replication = fs.getDefaultReplication(path);
  }
  if (blockSize == -1) {
    blockSize = fs.getDefaultBlockSize(path);
  }
  FsPermission fsPermission = new FsPermission(permission);
  int bufferSize = fs.getConf().getInt("httpfs.buffer.size", 4096);
  OutputStream os = fs.create(path, fsPermission, override, bufferSize, replication, blockSize, null);
  IOUtils.copyBytes(is, os, bufferSize, true);
  os.close();
  return null;
}
 

private static final OutputStream getOutputStream(FileSystem fileSystem, Path path) throws IOException {
  Configuration conf = fileSystem.getConf();
  FsServerDefaults fsDefaults = fileSystem.getServerDefaults(path);
  short replication = fileSystem.getDefaultReplication(path);
  EnumSet<CreateFlag> flags = EnumSet.of(CreateFlag.CREATE);
  if (Boolean.getBoolean(HDFS_SYNC_BLOCK)) {
    flags.add(CreateFlag.SYNC_BLOCK);
  }
  return fileSystem.create(path, FsPermission.getDefault()
      .applyUMask(FsPermission.getUMask(conf)), flags, fsDefaults
      .getFileBufferSize(), replication, fsDefaults
      .getBlockSize(), null);
}
 

public static void uploadSegment(StorageSegment segment,
                                 FileSystem fileSystem,
                                 Path path,
                                 Path tableLocation) throws IOException {
    short replica = fileSystem.getDefaultReplication(tableLocation);
    if (replica <= 0) {
        logger.warn("Failed to get replication from {}", tableLocation);
        replica = fileSystem.getDefaultReplication();
    }
    short _replica = replica;

    ByteBufferWriter.PredictSizeOpener writeOpener = size -> {
        long blockSize = getSegmentBlockSize(fileSystem, size);
        FSDataOutputStream outputStream = fileSystem.create(
                path,
                FsPermission.getFileDefault().applyUMask(FsPermission.getUMask(fileSystem.getConf())),
                EnumSet.of(CreateFlag.CREATE, CreateFlag.OVERWRITE),
                4096 * 10,
                _replica,
                blockSize,
                null);
        ByteBufferWriter writer = ByteBufferWriter.of(outputStream, outputStream::close);
        writer.setName(path.toString());
        return writer;
    };
    IntegratedSegment.Fd.create(segment, writeOpener, null);
}
 

public static void uploadSegment(StorageSegment segment,
                                 FileSystem fileSystem,
                                 Path path) throws IOException {
    Path parent = path.getParent();
    short replica = fileSystem.getDefaultReplication(parent);
    if (replica <= 0) {
        logger.warn("Failed to get replication from {}", parent);
        replica = fileSystem.getDefaultReplication();
    }
    short _replica = replica;

    ByteBufferWriter.PredictSizeOpener writeOpener = size -> {
        long blockSize = getSegmentBlockSize(fileSystem, size);
        FSDataOutputStream outputStream = fileSystem.create(
                path,
                FsPermission.getFileDefault().applyUMask(FsPermission.getUMask(fileSystem.getConf())),
                EnumSet.of(CreateFlag.CREATE, CreateFlag.OVERWRITE),
                4096 * 10,
                _replica,
                blockSize,
                null);
        ByteBufferWriter writer = ByteBufferWriter.of(outputStream, outputStream::close);
        writer.setName(path.toString());
        return writer;
    };
    IntegratedSegment.Fd.create(segment, writeOpener, null);
}
 

private static short getReplicationFactor(
        EnumSet<FileAttribute> fileAttributes,
        FileStatus sourceFile, FileSystem targetFS, Path tmpTargetPath) {
  return fileAttributes.contains(FileAttribute.REPLICATION)?
          sourceFile.getReplication() : targetFS.getDefaultReplication(tmpTargetPath);
}
 

Writer(Configuration conf, Option... opts) throws IOException {
  BlockSizeOption blockSizeOption =
      Options.getOption(BlockSizeOption.class, opts);
  BufferSizeOption bufferSizeOption =
      Options.getOption(BufferSizeOption.class, opts);
  ReplicationOption replicationOption =
      Options.getOption(ReplicationOption.class, opts);

  FileOption fileOption = Options.getOption(FileOption.class, opts);
  AppendIfExistsOption appendIfExistsOption = Options.getOption(
      AppendIfExistsOption.class, opts);
  StreamOption streamOption = Options.getOption(StreamOption.class, opts);

  // check consistency of options
  if ((fileOption == null) == (streamOption == null)) {
    throw new IllegalArgumentException("file or stream must be specified");
  }
  if (fileOption == null && (blockSizeOption != null ||
                             bufferSizeOption != null ||
                             replicationOption != null)) {
    throw new IllegalArgumentException("file modifier options not " +
                                       "compatible with stream");
  }

  FSDataOutputStream out;
  boolean ownStream = fileOption != null;
  if (ownStream) {
    Path p = fileOption.getValue();
    FileSystem fs;
    fs = p.getFileSystem(conf);
    int bufferSize = bufferSizeOption == null ? getBufferSize(conf) :
                     bufferSizeOption.getValue();
    short replication = replicationOption == null ?
                        fs.getDefaultReplication(p) :
                        (short) replicationOption.getValue();
    long blockSize = blockSizeOption == null ? fs.getDefaultBlockSize(p) :
                     blockSizeOption.getValue();

    if (appendIfExistsOption != null && appendIfExistsOption.getValue()
        && fs.exists(p)) {
      // Read the file and verify header details
      try (WALFile.Reader reader =
               new WALFile.Reader(conf, WALFile.Reader.file(p), new Reader.OnlyHeaderOption())){
        if (reader.getVersion() != VERSION[3]) {
          throw new VersionMismatchException(VERSION[3], reader.getVersion());
        }
        sync = reader.getSync();
      }
      out = fs.append(p, bufferSize);
      this.appendMode = true;
    } else {
      out = fs.create(p, true, bufferSize, replication, blockSize);
    }
  } else {
    out = streamOption.getValue();
  }

  init(conf, out, ownStream);
}
 

private static short getReplicationFactor(
        EnumSet<FileAttribute> fileAttributes,
        FileStatus sourceFile, FileSystem targetFS, Path tmpTargetPath) {
  return fileAttributes.contains(FileAttribute.REPLICATION)?
          sourceFile.getReplication() : targetFS.getDefaultReplication(tmpTargetPath);
}
 

public static Short getDefaultReplication(FileSystem fs, Path path) {
  return fs.getDefaultReplication(path);
}
 

public static SegmentFd uploadSegment(StorageSegment segment, FileSystem fileSystem, Path path, boolean openSegment) throws IOException {
    // First upload to a tmp file, then rename to real path after completely transfer all data, to avoid file corruption.

    Path tmpPath = new Path(path.toString() + ".__TMP__");
    SegmentFd fd;
    short replica = fileSystem.getDefaultReplication(path);
    if (replica <= 0) {
        logger.warn("Failed to get replication from {}", path);
        replica = fileSystem.getDefaultReplication();
    }
    short _replica = replica;

    ByteBufferWriter.PredictSizeOpener writeOpener = size -> {
        long blockSize = getSegmentBlockSize(fileSystem, size);
        FSDataOutputStream outputStream = fileSystem.create(
                tmpPath,
                FsPermission.getFileDefault().applyUMask(FsPermission.getUMask(fileSystem.getConf())),
                EnumSet.of(CreateFlag.CREATE, CreateFlag.OVERWRITE),
                4096 * 10,
                _replica,
                blockSize,
                null);
        ByteBufferWriter writer = ByteBufferWriter.of(outputStream, outputStream::close);
        writer.setName(tmpPath.toString());
        return writer;
    };
    ByteBufferReader.Opener readerOpener = openSegment ? ByteBufferReader.Opener.create(fileSystem, path) : null;
    fd = IntegratedSegment.Fd.create(segment, writeOpener, readerOpener);

    int times = 0;
    Boolean ok = false;
    while (times < 5) {
        if (fileSystem instanceof DistributedFileSystem) {
            ok = Try.on(() -> ((DistributedFileSystem) fileSystem).rename(tmpPath, path, Options.Rename.OVERWRITE), 1, logger);
        } else {
            ok = Try.on(() -> fileSystem.rename(tmpPath, path), 1, logger);
        }
        if (ok != null && ok) {
            break;
        }
        times++;
        if (fileSystem.exists(path)) {
            Try.on(() -> fileSystem.delete(path), 1, logger, String.format("delete old segment failed: %s", path.toString()));
        }
    }
    if (ok == null || !ok) {
        throw new IOException(String.format("Failed to rename from [%s] to [%s]", tmpPath, path));
    }

    return fd;
}
 

public static short getDefaultReplication(final FileSystem fs, final Path path) {
  return fs.getDefaultReplication(path);
}
 

/**
 * @return desired replication for destination file.
 */
public short getReplication(FileSystem targetFs) {
  return getPreserve().preserve(PreserveAttributes.Option.REPLICATION) ?
      getOrigin().getReplication() : targetFs.getDefaultReplication(this.destination);
}
 

public void checkFile(String cmd, String[] args, int startIndex)
    throws IOException {
  if (startIndex >= args.length) {
    printUsage(cmd);
    throw new IllegalArgumentException("Insufficient arguments");
  }
  for (int i = startIndex; i < args.length; i++) {
    Path p = new Path(args[i]);
    FileSystem fs = p.getFileSystem(conf);
    // if we got a raid fs, get the underlying fs 
    if (fs instanceof DistributedRaidFileSystem) {
      fs = ((DistributedRaidFileSystem) fs).getFileSystem();
    }
    // We should be able to cast at this point.
    DistributedFileSystem dfs = (DistributedFileSystem) fs;
    RemoteIterator<Path> corruptIt = dfs.listCorruptFileBlocks(p);
    int count = 0;
    while (corruptIt.hasNext()) {
      count++;
      Path corruptFile = corruptIt.next();
      // Result of checking.
      String result = null;
      FileStatus stat = fs.getFileStatus(p);
      if (stat.getReplication() < fs.getDefaultReplication()) {
        RaidInfo raidInfo = getFileRaidInfo(corruptFile);
        if (raidInfo.codec == null) {
          result = "Below default replication but no parity file found";
        } else {
          boolean notRecoverable = isFileCorrupt(dfs, corruptFile);
          if (notRecoverable) {
            result = "Missing too many blocks to be recovered " + 
              "using parity file " + raidInfo.parityPair.getPath();
          } else {
            result = "Has missing blocks but can be read using parity file " +
              raidInfo.parityPair.getPath();
          }
        }
      } else {
        result = "At default replication, not raided";
      }
      out.println("Result of checking " + corruptFile + " : " +
        result);
    }
    out.println("Found " + count + " files with missing blocks");
  }
}
 

/**
 * Constructs a RCFile Writer.
 *
 * @param fs
 *          the file system used
 * @param conf
 *          the configuration file
 * @param name
 *          the file name
 * @param progress a progress meter to update as the file is written
 * @param metadata a string to string map in the file header
 * @throws java.io.IOException
 */
public Writer(FileSystem fs, Configuration conf, Path name,
    Progressable progress, Metadata metadata, CompressionCodec codec) throws IOException {
  this(fs, conf, name, fs.getConf().getInt("io.file.buffer.size", 4096),
      fs.getDefaultReplication(), fs.getDefaultBlockSize(), progress,
      metadata, codec);
}