org.apache.hadoop.fs.Options.ChecksumOpt Java Exaples

Source File: FileSystem.java From big-c with Apache License 2.0

6 votes

@Deprecated
protected FSDataOutputStream primitiveCreate(Path f,
   FsPermission absolutePermission, EnumSet<CreateFlag> flag, int bufferSize,
   short replication, long blockSize, Progressable progress,
   ChecksumOpt checksumOpt) throws IOException {

  boolean pathExists = exists(f);
  CreateFlag.validate(f, pathExists, flag);
  
  // Default impl  assumes that permissions do not matter and 
  // nor does the bytesPerChecksum  hence
  // calling the regular create is good enough.
  // FSs that implement permissions should override this.

  if (pathExists && flag.contains(CreateFlag.APPEND)) {
    return append(f, bufferSize, progress);
  }
  
  return this.create(f, absolutePermission,
      flag.contains(CreateFlag.OVERWRITE), bufferSize, replication,
      blockSize, progress);
}

Source File: FileSystem.java From hadoop with Apache License 2.0

6 votes

@Deprecated
protected FSDataOutputStream primitiveCreate(Path f,
   FsPermission absolutePermission, EnumSet<CreateFlag> flag, int bufferSize,
   short replication, long blockSize, Progressable progress,
   ChecksumOpt checksumOpt) throws IOException {

  boolean pathExists = exists(f);
  CreateFlag.validate(f, pathExists, flag);
  
  // Default impl  assumes that permissions do not matter and 
  // nor does the bytesPerChecksum  hence
  // calling the regular create is good enough.
  // FSs that implement permissions should override this.

  if (pathExists && flag.contains(CreateFlag.APPEND)) {
    return append(f, bufferSize, progress);
  }
  
  return this.create(f, absolutePermission,
      flag.contains(CreateFlag.OVERWRITE), bufferSize, replication,
      blockSize, progress);
}

Source File: ViewFs.java From hadoop with Apache License 2.0

6 votes

@Override
public FSDataOutputStream createInternal(final Path f,
    final EnumSet<CreateFlag> flag, final FsPermission absolutePermission,
    final int bufferSize, final short replication, final long blockSize,
    final Progressable progress, final ChecksumOpt checksumOpt,
    final boolean createParent) throws AccessControlException,
    FileAlreadyExistsException, FileNotFoundException,
    ParentNotDirectoryException, UnsupportedFileSystemException,
    UnresolvedLinkException, IOException {
  InodeTree.ResolveResult<AbstractFileSystem> res;
  try {
    res = fsState.resolve(getUriPath(f), false);
  } catch (FileNotFoundException e) {
    if (createParent) {
      throw readOnlyMountTable("create", f);
    } else {
      throw e;
    }
  }
  assert(res.remainingPath != null);
  return res.targetFileSystem.createInternal(res.remainingPath, flag,
      absolutePermission, bufferSize, replication,
      blockSize, progress, checksumOpt,
      createParent);
}

Source File: ViewFs.java From big-c with Apache License 2.0

6 votes

@Override
public FSDataOutputStream createInternal(final Path f,
    final EnumSet<CreateFlag> flag, final FsPermission absolutePermission,
    final int bufferSize, final short replication, final long blockSize,
    final Progressable progress, final ChecksumOpt checksumOpt,
    final boolean createParent) throws AccessControlException,
    FileAlreadyExistsException, FileNotFoundException,
    ParentNotDirectoryException, UnsupportedFileSystemException,
    UnresolvedLinkException, IOException {
  InodeTree.ResolveResult<AbstractFileSystem> res;
  try {
    res = fsState.resolve(getUriPath(f), false);
  } catch (FileNotFoundException e) {
    if (createParent) {
      throw readOnlyMountTable("create", f);
    } else {
      throw e;
    }
  }
  assert(res.remainingPath != null);
  return res.targetFileSystem.createInternal(res.remainingPath, flag,
      absolutePermission, bufferSize, replication,
      blockSize, progress, checksumOpt,
      createParent);
}

Source File: DFSClient.java From hadoop with Apache License 2.0

6 votes

/**
 * Same as {@link #create(String, FsPermission, EnumSet, boolean, short, long,
 * Progressable, int, ChecksumOpt)} with the addition of favoredNodes that is
 * a hint to where the namenode should place the file blocks.
 * The favored nodes hint is not persisted in HDFS. Hence it may be honored
 * at the creation time only. HDFS could move the blocks during balancing or
 * replication, to move the blocks from favored nodes. A value of null means
 * no favored nodes for this create
 */
public DFSOutputStream create(String src, 
                           FsPermission permission,
                           EnumSet<CreateFlag> flag, 
                           boolean createParent,
                           short replication,
                           long blockSize,
                           Progressable progress,
                           int buffersize,
                           ChecksumOpt checksumOpt,
                           InetSocketAddress[] favoredNodes) throws IOException {
  checkOpen();
  if (permission == null) {
    permission = FsPermission.getFileDefault();
  }
  FsPermission masked = permission.applyUMask(dfsClientConf.uMask);
  if(LOG.isDebugEnabled()) {
    LOG.debug(src + ": masked=" + masked);
  }
  final DFSOutputStream result = DFSOutputStream.newStreamForCreate(this,
      src, masked, flag, createParent, replication, blockSize, progress,
      buffersize, dfsClientConf.createChecksum(checksumOpt),
      getFavoredNodesStr(favoredNodes));
  beginFileLease(result.getFileId(), result);
  return result;
}

Source File: DFSClient.java From hadoop with Apache License 2.0

6 votes

/**
 * Same as {{@link #create(String, FsPermission, EnumSet, short, long,
 *  Progressable, int, ChecksumOpt)} except that the permission
 *  is absolute (ie has already been masked with umask.
 */
public DFSOutputStream primitiveCreate(String src, 
                           FsPermission absPermission,
                           EnumSet<CreateFlag> flag,
                           boolean createParent,
                           short replication,
                           long blockSize,
                           Progressable progress,
                           int buffersize,
                           ChecksumOpt checksumOpt)
    throws IOException, UnresolvedLinkException {
  checkOpen();
  CreateFlag.validate(flag);
  DFSOutputStream result = primitiveAppend(src, flag, buffersize, progress);
  if (result == null) {
    DataChecksum checksum = dfsClientConf.createChecksum(checksumOpt);
    result = DFSOutputStream.newStreamForCreate(this, src, absPermission,
        flag, createParent, replication, blockSize, progress, buffersize,
        checksum, null);
  }
  beginFileLease(result.getFileId(), result);
  return result;
}

Source File: DFSClient.java From big-c with Apache License 2.0

6 votes

/**
 * Same as {@link #create(String, FsPermission, EnumSet, boolean, short, long,
 * Progressable, int, ChecksumOpt)} with the addition of favoredNodes that is
 * a hint to where the namenode should place the file blocks.
 * The favored nodes hint is not persisted in HDFS. Hence it may be honored
 * at the creation time only. HDFS could move the blocks during balancing or
 * replication, to move the blocks from favored nodes. A value of null means
 * no favored nodes for this create
 */
public DFSOutputStream create(String src, 
                           FsPermission permission,
                           EnumSet<CreateFlag> flag, 
                           boolean createParent,
                           short replication,
                           long blockSize,
                           Progressable progress,
                           int buffersize,
                           ChecksumOpt checksumOpt,
                           InetSocketAddress[] favoredNodes) throws IOException {
  checkOpen();
  if (permission == null) {
    permission = FsPermission.getFileDefault();
  }
  FsPermission masked = permission.applyUMask(dfsClientConf.uMask);
  if(LOG.isDebugEnabled()) {
    LOG.debug(src + ": masked=" + masked);
  }
  final DFSOutputStream result = DFSOutputStream.newStreamForCreate(this,
      src, masked, flag, createParent, replication, blockSize, progress,
      buffersize, dfsClientConf.createChecksum(checksumOpt),
      getFavoredNodesStr(favoredNodes));
  beginFileLease(result.getFileId(), result);
  return result;
}

Source File: DFSClient.java From big-c with Apache License 2.0

6 votes

/**
 * Same as {{@link #create(String, FsPermission, EnumSet, short, long,
 *  Progressable, int, ChecksumOpt)} except that the permission
 *  is absolute (ie has already been masked with umask.
 */
public DFSOutputStream primitiveCreate(String src, 
                           FsPermission absPermission,
                           EnumSet<CreateFlag> flag,
                           boolean createParent,
                           short replication,
                           long blockSize,
                           Progressable progress,
                           int buffersize,
                           ChecksumOpt checksumOpt)
    throws IOException, UnresolvedLinkException {
  checkOpen();
  CreateFlag.validate(flag);
  DFSOutputStream result = primitiveAppend(src, flag, buffersize, progress);
  if (result == null) {
    DataChecksum checksum = dfsClientConf.createChecksum(checksumOpt);
    result = DFSOutputStream.newStreamForCreate(this, src, absPermission,
        flag, createParent, replication, blockSize, progress, buffersize,
        checksum, null);
  }
  beginFileLease(result.getFileId(), result);
  return result;
}

Source File: FilterFileSystem.java From big-c with Apache License 2.0

5 votes

@Override
protected FSDataOutputStream primitiveCreate(Path f,
    FsPermission absolutePermission, EnumSet<CreateFlag> flag,
    int bufferSize, short replication, long blockSize,
    Progressable progress, ChecksumOpt checksumOpt)
    throws IOException {
  return fs.primitiveCreate(f, absolutePermission, flag,
      bufferSize, replication, blockSize, progress, checksumOpt);
}

Source File: TestAfsCheckPath.java From big-c with Apache License 2.0

5 votes

@Override
public FSDataOutputStream createInternal(Path f, EnumSet<CreateFlag> flag,
    FsPermission absolutePermission, int bufferSize, short replication,
    long blockSize, Progressable progress, ChecksumOpt checksumOpt,
    boolean createParent) throws IOException {
  // deliberately empty
  return null;
}

Source File: TestFilterFileSystem.java From big-c with Apache License 2.0

5 votes

public FSDataOutputStream create(Path f,
    FsPermission permission,
    EnumSet<CreateFlag> flags,
    int bufferSize,
    short replication,
    long blockSize,
    Progressable progress,
    ChecksumOpt checksumOpt) throws IOException {
  return null;
}

Source File: AbstractFileSystem.java From big-c with Apache License 2.0

5 votes

/**
 * The specification of this method matches that of
 * {@link #create(Path, EnumSet, Options.CreateOpts...)} except that the opts
 * have been declared explicitly.
 */
public abstract FSDataOutputStream createInternal(Path f,
    EnumSet<CreateFlag> flag, FsPermission absolutePermission,
    int bufferSize, short replication, long blockSize, Progressable progress,
    ChecksumOpt checksumOpt, boolean createParent)
    throws AccessControlException, FileAlreadyExistsException,
    FileNotFoundException, ParentNotDirectoryException,
    UnsupportedFileSystemException, UnresolvedLinkException, IOException;

Source File: ViewFs.java From big-c with Apache License 2.0

5 votes

@Override
public FSDataOutputStream createInternal(final Path f,
    final EnumSet<CreateFlag> flag, final FsPermission absolutePermission,
    final int bufferSize, final short replication, final long blockSize,
    final Progressable progress, final ChecksumOpt checksumOpt,
    final boolean createParent) throws AccessControlException,
    FileAlreadyExistsException, FileNotFoundException,
    ParentNotDirectoryException, UnsupportedFileSystemException,
    UnresolvedLinkException, IOException {
  throw readOnlyMountTable("create", f);
}

Source File: ChecksumFs.java From big-c with Apache License 2.0

5 votes

public ChecksumFSOutputSummer(final ChecksumFs fs, final Path file, 
  final EnumSet<CreateFlag> createFlag,
  final FsPermission absolutePermission, final int bufferSize,
  final short replication, final long blockSize, 
  final Progressable progress, final ChecksumOpt checksumOpt,
  final boolean createParent) throws IOException {
  super(DataChecksum.newDataChecksum(DataChecksum.Type.CRC32,
      fs.getBytesPerSum()));

  // checksumOpt is passed down to the raw fs. Unless it implements
  // checksum impelemts internally, checksumOpt will be ignored.
  // If the raw fs does checksum internally, we will end up with
  // two layers of checksumming. i.e. checksumming checksum file.
  this.datas = fs.getRawFs().createInternal(file, createFlag,
      absolutePermission, bufferSize, replication, blockSize, progress,
       checksumOpt,  createParent);
  
  // Now create the chekcsumfile; adjust the buffsize
  int bytesPerSum = fs.getBytesPerSum();
  int sumBufferSize = fs.getSumBufferSize(bytesPerSum, bufferSize);
  this.sums = fs.getRawFs().createInternal(fs.getChecksumFile(file),
      EnumSet.of(CreateFlag.CREATE, CreateFlag.OVERWRITE),
      absolutePermission, sumBufferSize, replication, blockSize, progress,
      checksumOpt, createParent);
  sums.write(CHECKSUM_VERSION, 0, CHECKSUM_VERSION.length);
  sums.writeInt(bytesPerSum);
}

Source File: TestCopyMapper.java From big-c with Apache License 2.0

5 votes

private static void touchFile(String path, boolean createMultipleBlocks,
    ChecksumOpt checksumOpt) throws Exception {
  FileSystem fs;
  DataOutputStream outputStream = null;
  try {
    fs = cluster.getFileSystem();
    final Path qualifiedPath = new Path(path).makeQualified(fs.getUri(),
        fs.getWorkingDirectory());
    final long blockSize = createMultipleBlocks ? NON_DEFAULT_BLOCK_SIZE : fs
        .getDefaultBlockSize(qualifiedPath) * 2;
    FsPermission permission = FsPermission.getFileDefault().applyUMask(
        FsPermission.getUMask(fs.getConf()));
    outputStream = fs.create(qualifiedPath, permission,
        EnumSet.of(CreateFlag.CREATE, CreateFlag.OVERWRITE), 0,
        (short) (fs.getDefaultReplication(qualifiedPath) * 2), blockSize,
        null, checksumOpt);
    byte[] bytes = new byte[DEFAULT_FILE_SIZE];
    outputStream.write(bytes);
    long fileSize = DEFAULT_FILE_SIZE;
    if (createMultipleBlocks) {
      while (fileSize < 2*blockSize) {
        outputStream.write(bytes);
        outputStream.flush();
        fileSize += DEFAULT_FILE_SIZE;
      }
    }
    pathList.add(qualifiedPath);
    ++nFiles;

    FileStatus fileStatus = fs.getFileStatus(qualifiedPath);
    System.out.println(fileStatus.getBlockSize());
    System.out.println(fileStatus.getReplication());
  }
  finally {
    IOUtils.cleanup(null, outputStream);
  }
}

Source File: FilterFs.java From big-c with Apache License 2.0

5 votes

@Override
public FSDataOutputStream createInternal(Path f,
  EnumSet<CreateFlag> flag, FsPermission absolutePermission, int bufferSize,
  short replication, long blockSize, Progressable progress,
  ChecksumOpt checksumOpt, boolean createParent) 
    throws IOException, UnresolvedLinkException {
  checkPath(f);
  return myFs.createInternal(f, flag, absolutePermission, bufferSize,
      replication, blockSize, progress, checksumOpt, createParent);
}

Source File: HadoopFileSystemWrapper.java From dremio-oss with Apache License 2.0

5 votes

@Override
public FSDataOutputStream create(Path f, FsPermission permission, EnumSet<CreateFlag> flags, int bufferSize,
    short replication, long blockSize, Progressable progress, ChecksumOpt checksumOpt) throws IOException {
  try (WaitRecorder recorder = OperatorStats.getWaitRecorder(operatorStats)) {
    return newFSDataOutputStreamWrapper(underlyingFs.create(f, permission, flags, bufferSize, replication,
        blockSize, progress, checksumOpt));
  } catch(FSError e) {
    throw propagateFSError(e);
  }
}

Source File: ChecksumFs.java From big-c with Apache License 2.0

5 votes

@Override
public FSDataOutputStream createInternal(Path f,
    EnumSet<CreateFlag> createFlag, FsPermission absolutePermission,
    int bufferSize, short replication, long blockSize, Progressable progress,
    ChecksumOpt checksumOpt, boolean createParent) throws IOException {
  final FSDataOutputStream out = new FSDataOutputStream(
      new ChecksumFSOutputSummer(this, f, createFlag, absolutePermission,
          bufferSize, replication, blockSize, progress,
          checksumOpt,  createParent), null);
  return out;
}

Source File: DelegateToFileSystem.java From big-c with Apache License 2.0

5 votes

@Override
@SuppressWarnings("deprecation") // call to primitiveCreate
public FSDataOutputStream createInternal (Path f,
    EnumSet<CreateFlag> flag, FsPermission absolutePermission, int bufferSize,
    short replication, long blockSize, Progressable progress,
    ChecksumOpt checksumOpt, boolean createParent) throws IOException {
  checkPath(f);
  
  // Default impl assumes that permissions do not matter
  // calling the regular create is good enough.
  // FSs that implement permissions should override this.

  if (!createParent) { // parent must exist.
    // since this.create makes parent dirs automatically
    // we must throw exception if parent does not exist.
    final FileStatus stat = getFileStatus(f.getParent());
    if (stat == null) {
      throw new FileNotFoundException("Missing parent:" + f);
    }
    if (!stat.isDirectory()) {
        throw new ParentNotDirectoryException("parent is not a dir:" + f);
    }
    // parent does exist - go ahead with create of file.
  }
  return fsImpl.primitiveCreate(f, absolutePermission, flag,
      bufferSize, replication, blockSize, progress, checksumOpt);
}

Source File: GoogleHadoopFS.java From hadoop-connectors with Apache License 2.0

5 votes

@Override
public FSDataOutputStream createInternal(
    Path file,
    EnumSet<CreateFlag> flag,
    FsPermission absolutePermission,
    int bufferSize,
    short replication,
    long blockSize,
    Progressable progress,
    ChecksumOpt checksumOpt,
    boolean createParent) throws IOException {
  logger.atFinest().log(
      "createInternal(file: %s, flag: %s, absolutePermission: %s, bufferSize: %d, "
          + "replication: %b, blockSize: %d, progress: %s, checksumOpt: %s, createParent: %b)",
      file,
      flag,
      absolutePermission,
      bufferSize,
      replication,
      blockSize,
      progress,
      checksumOpt,
      createParent);
  if (!createParent) {
    // TODO: don't ignore 'createParent' flag
    logger.atFine().log("Ignoring createParent=false. Creating parents anyways.");
  }
  // AbstractFileSystems rely on permission to not overwrite.
  boolean overwriteFile = true;
  return ghfs.create(
      file, absolutePermission, overwriteFile, bufferSize, replication, blockSize, progress);
}

Source File: ChRootedFs.java From big-c with Apache License 2.0

5 votes

@Override
public FSDataOutputStream createInternal(final Path f,
    final EnumSet<CreateFlag> flag, final FsPermission absolutePermission,
    final int bufferSize, final short replication, final long blockSize,
    final Progressable progress, final ChecksumOpt checksumOpt,
    final boolean createParent) throws IOException, UnresolvedLinkException {
  return myFs.createInternal(fullPath(f), flag,
      absolutePermission, bufferSize,
      replication, blockSize, progress, checksumOpt, createParent);
}

Source File: FilterFileSystem.java From big-c with Apache License 2.0

5 votes

@Override
public FSDataOutputStream create(Path f,
      FsPermission permission,
      EnumSet<CreateFlag> flags,
      int bufferSize,
      short replication,
      long blockSize,
      Progressable progress,
      ChecksumOpt checksumOpt) throws IOException {
  return fs.create(f, permission,
    flags, bufferSize, replication, blockSize, progress, checksumOpt);
}

Source File: TestCopyMapper.java From big-c with Apache License 2.0

5 votes

private static void createSourceDataWithDifferentChecksumType()
    throws Exception {
  mkdirs(SOURCE_PATH + "/1");
  mkdirs(SOURCE_PATH + "/2");
  mkdirs(SOURCE_PATH + "/2/3/4");
  mkdirs(SOURCE_PATH + "/2/3");
  mkdirs(SOURCE_PATH + "/5");
  touchFile(SOURCE_PATH + "/5/6", new ChecksumOpt(DataChecksum.Type.CRC32,
      512));
  mkdirs(SOURCE_PATH + "/7");
  mkdirs(SOURCE_PATH + "/7/8");
  touchFile(SOURCE_PATH + "/7/8/9", new ChecksumOpt(DataChecksum.Type.CRC32C,
      512));
}

Source File: RetriableFileCopyCommand.java From big-c with Apache License 2.0

5 votes

/**
 * @return the checksum spec of the source checksum if checksum type should be
 *         preserved
 */
private ChecksumOpt getChecksumOpt(EnumSet<FileAttribute> fileAttributes,
    FileChecksum sourceChecksum) {
  if (fileAttributes.contains(FileAttribute.CHECKSUMTYPE)
      && sourceChecksum != null) {
    return sourceChecksum.getChecksumOpt();
  }
  return null;
}

Source File: DistributedFileSystem.java From big-c with Apache License 2.0

5 votes

@Override
protected HdfsDataOutputStream primitiveCreate(Path f,
  FsPermission absolutePermission, EnumSet<CreateFlag> flag, int bufferSize,
  short replication, long blockSize, Progressable progress,
  ChecksumOpt checksumOpt) throws IOException {
  statistics.incrementWriteOps(1);
  final DFSOutputStream dfsos = dfs.primitiveCreate(
    getPathName(fixRelativePart(f)),
    absolutePermission, flag, true, replication, blockSize,
    progress, bufferSize, checksumOpt);
  return dfs.createWrappedOutputStream(dfsos, statistics);
}

Source File: DFSClient.java From big-c with Apache License 2.0

5 votes

/**
 * Call {@link #create(String, FsPermission, EnumSet, boolean, short, 
 * long, Progressable, int, ChecksumOpt)} with <code>createParent</code>
 *  set to true.
 */
public DFSOutputStream create(String src, 
                           FsPermission permission,
                           EnumSet<CreateFlag> flag, 
                           short replication,
                           long blockSize,
                           Progressable progress,
                           int buffersize,
                           ChecksumOpt checksumOpt)
    throws IOException {
  return create(src, permission, flag, true,
      replication, blockSize, progress, buffersize, checksumOpt, null);
}

Source File: DFSClient.java From big-c with Apache License 2.0

5 votes

private DataChecksum createChecksum(ChecksumOpt userOpt) {
  // Fill in any missing field with the default.
  ChecksumOpt myOpt = ChecksumOpt.processChecksumOpt(
      defaultChecksumOpt, userOpt);
  DataChecksum dataChecksum = DataChecksum.newDataChecksum(
      myOpt.getChecksumType(),
      myOpt.getBytesPerChecksum());
  if (dataChecksum == null) {
    throw new HadoopIllegalArgumentException("Invalid checksum type: userOpt="
        + userOpt + ", default=" + defaultChecksumOpt
        + ", effective=null");
  }
  return dataChecksum;
}

Source File: Hdfs.java From big-c with Apache License 2.0

5 votes

@Override
public HdfsDataOutputStream createInternal(Path f,
    EnumSet<CreateFlag> createFlag, FsPermission absolutePermission,
    int bufferSize, short replication, long blockSize, Progressable progress,
    ChecksumOpt checksumOpt, boolean createParent) throws IOException {

  final DFSOutputStream dfsos = dfs.primitiveCreate(getUriPath(f),
    absolutePermission, createFlag, createParent, replication, blockSize,
    progress, bufferSize, checksumOpt);
  return dfs.createWrappedOutputStream(dfsos, statistics,
      dfsos.getInitialLen());
}

Source File: TestAfsCheckPath.java From hadoop with Apache License 2.0

5 votes

@Override
public FSDataOutputStream createInternal(Path f, EnumSet<CreateFlag> flag,
    FsPermission absolutePermission, int bufferSize, short replication,
    long blockSize, Progressable progress, ChecksumOpt checksumOpt,
    boolean createParent) throws IOException {
  // deliberately empty
  return null;
}

Source File: TestFilterFileSystem.java From hadoop with Apache License 2.0

5 votes

public FSDataOutputStream create(Path f,
    FsPermission permission,
    EnumSet<CreateFlag> flags,
    int bufferSize,
    short replication,
    long blockSize,
    Progressable progress,
    ChecksumOpt checksumOpt) throws IOException {
  return null;
}

org.apache.hadoop.fs.Options.ChecksumOpt Java Examples