org.apache.hadoop.hdfs.client.HdfsDataInputStream Java Examples

public HdfsDataInputStream createWrappedInputStream(DFSInputStream dfsis)
        throws IOException {
    final FileEncryptionInfo feInfo = dfsis.getFileEncryptionInfo();
    if (feInfo != null) {
        // File is encrypted, wrap the stream in a crypto stream.
        // Currently only one version, so no special logic based on the version #
        getCryptoProtocolVersion(feInfo);
        final CryptoCodec codec = getCryptoCodec(getConfiguration(), feInfo);
        final KeyProvider.KeyVersion decrypted = decryptEncryptedDataEncryptionKey(dfsis, feInfo);
        final CryptoInputStream cryptoIn =
                new CryptoInputStream(dfsis, codec, decrypted.getMaterial(),
                        feInfo.getIV());
        return new HdfsDataInputStream(cryptoIn);
    } else {
        // No FileEncryptionInfo so no encryption.
        return new HdfsDataInputStream(dfsis);
    }
}
 

/**
 * Wraps the stream in a CryptoInputStream if the underlying file is
 * encrypted.
 */
public HdfsDataInputStream createWrappedInputStream(DFSInputStream dfsis)
    throws IOException {
  final FileEncryptionInfo feInfo = dfsis.getFileEncryptionInfo();
  if (feInfo != null) {
    // File is encrypted, wrap the stream in a crypto stream.
    // Currently only one version, so no special logic based on the version #
    getCryptoProtocolVersion(feInfo);
    final CryptoCodec codec = getCryptoCodec(conf, feInfo);
    final KeyVersion decrypted = decryptEncryptedDataEncryptionKey(feInfo);
    final CryptoInputStream cryptoIn =
        new CryptoInputStream(dfsis, codec, decrypted.getMaterial(),
            feInfo.getIV());
    return new HdfsDataInputStream(cryptoIn);
  } else {
    // No FileEncryptionInfo so no encryption.
    return new HdfsDataInputStream(dfsis);
  }
}
 

private void updateInputStreamStatistics(FSDataInputStream stream) {
  // If the underlying file system is HDFS, update read statistics upon close.
  if (stream instanceof HdfsDataInputStream) {
    /**
     * Because HDFS ReadStatistics is calculated per input stream, it is not
     * feasible to update the aggregated number in real time. Instead, the
     * metrics are updated when an input stream is closed.
     */
    HdfsDataInputStream hdfsDataInputStream = (HdfsDataInputStream)stream;
    synchronized (readStatistics) {
      readStatistics.totalBytesRead += hdfsDataInputStream.getReadStatistics().
        getTotalBytesRead();
      readStatistics.totalLocalBytesRead += hdfsDataInputStream.getReadStatistics().
        getTotalLocalBytesRead();
      readStatistics.totalShortCircuitBytesRead += hdfsDataInputStream.getReadStatistics().
        getTotalShortCircuitBytesRead();
      readStatistics.totalZeroCopyBytesRead += hdfsDataInputStream.getReadStatistics().
        getTotalZeroCopyBytesRead();
    }
  }
}
 

static void checkFile(Path p, int expectedsize, final Configuration conf
    ) throws IOException, InterruptedException {
  //open the file with another user account
  final String username = UserGroupInformation.getCurrentUser().getShortUserName()
      + "_" + ++userCount;

  UserGroupInformation ugi = UserGroupInformation.createUserForTesting(username, 
                               new String[] {"supergroup"});
  
  final FileSystem fs = DFSTestUtil.getFileSystemAs(ugi, conf);
  
  final HdfsDataInputStream in = (HdfsDataInputStream)fs.open(p);

  //Check visible length
  Assert.assertTrue(in.getVisibleLength() >= expectedsize);

  //Able to read?
  for(int i = 0; i < expectedsize; i++) {
    Assert.assertEquals((byte)i, (byte)in.read());  
  }

  in.close();
}
 

static void checkFile(Path p, int expectedsize, final Configuration conf
    ) throws IOException, InterruptedException {
  //open the file with another user account
  final String username = UserGroupInformation.getCurrentUser().getShortUserName()
      + "_" + ++userCount;

  UserGroupInformation ugi = UserGroupInformation.createUserForTesting(username, 
                               new String[] {"supergroup"});
  
  final FileSystem fs = DFSTestUtil.getFileSystemAs(ugi, conf);
  
  final HdfsDataInputStream in = (HdfsDataInputStream)fs.open(p);

  //Check visible length
  Assert.assertTrue(in.getVisibleLength() >= expectedsize);

  //Able to read?
  for(int i = 0; i < expectedsize; i++) {
    Assert.assertEquals((byte)i, (byte)in.read());  
  }

  in.close();
}
 

/**
 * Wraps the stream in a CryptoInputStream if the underlying file is
 * encrypted.
 */
public HdfsDataInputStream createWrappedInputStream(DFSInputStream dfsis)
    throws IOException {
  final FileEncryptionInfo feInfo = dfsis.getFileEncryptionInfo();
  if (feInfo != null) {
    // File is encrypted, wrap the stream in a crypto stream.
    // Currently only one version, so no special logic based on the version #
    getCryptoProtocolVersion(feInfo);
    final CryptoCodec codec = getCryptoCodec(conf, feInfo);
    final KeyVersion decrypted = decryptEncryptedDataEncryptionKey(feInfo);
    final CryptoInputStream cryptoIn =
        new CryptoInputStream(dfsis, codec, decrypted.getMaterial(),
            feInfo.getIV());
    return new HdfsDataInputStream(cryptoIn);
  } else {
    // No FileEncryptionInfo so no encryption.
    return new HdfsDataInputStream(dfsis);
  }
}
 

@SuppressWarnings("deprecation")
@Override
public HdfsDataInputStream open(Path f, int bufferSize) 
    throws IOException, UnresolvedLinkException {
  final DFSInputStream dfsis = dfs.open(getUriPath(f),
    bufferSize, verifyChecksum);
  return dfs.createWrappedInputStream(dfsis);
}
 

/**
 * Read in-memory block with Short Circuit Read
 * Note: the test uses faked RAM_DISK from physical disk.
 */
@Test
public void testRamDiskShortCircuitRead()
  throws IOException, InterruptedException {
  startUpCluster(REPL_FACTOR,
    new StorageType[]{RAM_DISK, DEFAULT},
    2 * BLOCK_SIZE - 1, true);  // 1 replica + delta, SCR read
  final String METHOD_NAME = GenericTestUtils.getMethodName();
  final int SEED = 0xFADED;
  Path path = new Path("/" + METHOD_NAME + ".dat");

  makeRandomTestFile(path, BLOCK_SIZE, true, SEED);
  ensureFileReplicasOnStorageType(path, RAM_DISK);

  // Sleep for a short time to allow the lazy writer thread to do its job
  Thread.sleep(3 * LAZY_WRITER_INTERVAL_SEC * 1000);

  //assertThat(verifyReadRandomFile(path, BLOCK_SIZE, SEED), is(true));
  FSDataInputStream fis = fs.open(path);

  // Verify SCR read counters
  try {
    fis = fs.open(path);
    byte[] buf = new byte[BUFFER_LENGTH];
    fis.read(0, buf, 0, BUFFER_LENGTH);
    HdfsDataInputStream dfsis = (HdfsDataInputStream) fis;
    Assert.assertEquals(BUFFER_LENGTH,
      dfsis.getReadStatistics().getTotalBytesRead());
    Assert.assertEquals(BUFFER_LENGTH,
      dfsis.getReadStatistics().getTotalShortCircuitBytesRead());
  } finally {
    fis.close();
    fis = null;
  }
}
 

private boolean checkUnsupportedMethod(FileSystem fs, Path file,
                                         byte[] expected, int readOffset) throws IOException {
  HdfsDataInputStream stm = (HdfsDataInputStream)fs.open(file);
  ByteBuffer actual = ByteBuffer.allocateDirect(expected.length - readOffset);
  IOUtils.skipFully(stm, readOffset);
  try {
    stm.read(actual);
  } catch(UnsupportedOperationException unex) {
    return true;
  }
  return false;
}
 

/**
 * Open the file to read from begin to end. Then close the file. 
 * Return number of bytes read. 
 * Support both sequential read and position read.
 */
private long readData(String fname, byte[] buffer, long byteExpected, long beginPosition)
    throws IOException {
  long totalByteRead = 0;
  Path path = getFullyQualifiedPath(fname);

  FSDataInputStream in = null;
  try {
    in = openInputStream(path);

    long visibleLenFromReadStream = ((HdfsDataInputStream)in).getVisibleLength();

    if (visibleLenFromReadStream < byteExpected)
    {
      throw new IOException(visibleLenFromReadStream
          + " = visibleLenFromReadStream < bytesExpected= "
          + byteExpected);
    }

    totalByteRead = readUntilEnd(in, buffer, buffer.length, fname,
        beginPosition, visibleLenFromReadStream, positionReadOption);
    in.close();

    // reading more data than visibleLeng is OK, but not less
    if (totalByteRead + beginPosition < byteExpected ){
      throw new IOException("readData mismatch in byte read: expected=" 
          + byteExpected + " ; got " +  (totalByteRead + beginPosition));
    }
    return totalByteRead + beginPosition;

  } catch (IOException e) {
    throw new IOException("##### Caught Exception in readData. "
        + "Total Byte Read so far = " + totalByteRead + " beginPosition = "
        + beginPosition, e);
  } finally {
    if (in != null)
      in.close();
  }
}
 

public static ExtendedBlock getFirstBlock(FileSystem fs, Path path) throws IOException {
  HdfsDataInputStream in = (HdfsDataInputStream) fs.open(path);
  try {
    in.readByte();
    return in.getCurrentBlock();
  } finally {
    in.close();
  }
}
 

private void onOpen(ChannelHandlerContext ctx) throws IOException {
  final String nnId = params.namenodeId();
  final int bufferSize = params.bufferSize();
  final long offset = params.offset();
  final long length = params.length();

  DefaultHttpResponse response = new DefaultHttpResponse(HTTP_1_1, OK);
  HttpHeaders headers = response.headers();
  // Allow the UI to access the file
  headers.set(ACCESS_CONTROL_ALLOW_METHODS, GET);
  headers.set(ACCESS_CONTROL_ALLOW_ORIGIN, "*");
  headers.set(CONTENT_TYPE, APPLICATION_OCTET_STREAM);
  headers.set(CONNECTION, CLOSE);

  final DFSClient dfsclient = newDfsClient(nnId, conf);
  HdfsDataInputStream in = dfsclient.createWrappedInputStream(
    dfsclient.open(path, bufferSize, true));
  in.seek(offset);

  long contentLength = in.getVisibleLength() - offset;
  if (length >= 0) {
    contentLength = Math.min(contentLength, length);
  }
  final InputStream data;
  if (contentLength >= 0) {
    headers.set(CONTENT_LENGTH, contentLength);
    data = new LimitInputStream(in, contentLength);
  } else {
    data = in;
  }

  ctx.write(response);
  ctx.writeAndFlush(new ChunkedStream(data) {
    @Override
    public void close() throws Exception {
      super.close();
      dfsclient.close();
    }
  }).addListener(ChannelFutureListener.CLOSE);
}
 

/**
 * Open the file to read from begin to end. Then close the file. 
 * Return number of bytes read. 
 * Support both sequential read and position read.
 */
private long readData(String fname, byte[] buffer, long byteExpected, long beginPosition)
    throws IOException {
  long totalByteRead = 0;
  Path path = getFullyQualifiedPath(fname);

  FSDataInputStream in = null;
  try {
    in = openInputStream(path);

    long visibleLenFromReadStream = ((HdfsDataInputStream)in).getVisibleLength();

    if (visibleLenFromReadStream < byteExpected)
    {
      throw new IOException(visibleLenFromReadStream
          + " = visibleLenFromReadStream < bytesExpected= "
          + byteExpected);
    }

    totalByteRead = readUntilEnd(in, buffer, buffer.length, fname,
        beginPosition, visibleLenFromReadStream, positionReadOption);
    in.close();

    // reading more data than visibleLeng is OK, but not less
    if (totalByteRead + beginPosition < byteExpected ){
      throw new IOException("readData mismatch in byte read: expected=" 
          + byteExpected + " ; got " +  (totalByteRead + beginPosition));
    }
    return totalByteRead + beginPosition;

  } catch (IOException e) {
    throw new IOException("##### Caught Exception in readData. "
        + "Total Byte Read so far = " + totalByteRead + " beginPosition = "
        + beginPosition, e);
  } finally {
    if (in != null)
      in.close();
  }
}
 

@SuppressWarnings("deprecation")
@Override
public HdfsDataInputStream open(Path f, int bufferSize) 
    throws IOException, UnresolvedLinkException {
  final DFSInputStream dfsis = dfs.open(getUriPath(f),
    bufferSize, verifyChecksum);
  return dfs.createWrappedInputStream(dfsis);
}
 

private void onOpen(ChannelHandlerContext ctx) throws IOException {
  final String nnId = params.namenodeId();
  final int bufferSize = params.bufferSize();
  final long offset = params.offset();
  final long length = params.length();

  DefaultHttpResponse response = new DefaultHttpResponse(HTTP_1_1, OK);
  HttpHeaders headers = response.headers();
  // Allow the UI to access the file
  headers.set(ACCESS_CONTROL_ALLOW_METHODS, GET);
  headers.set(ACCESS_CONTROL_ALLOW_ORIGIN, "*");
  headers.set(CONTENT_TYPE, APPLICATION_OCTET_STREAM);
  headers.set(CONNECTION, CLOSE);

  final DFSClient dfsclient = newDfsClient(nnId, conf);
  HdfsDataInputStream in = dfsclient.createWrappedInputStream(
    dfsclient.open(path, bufferSize, true));
  in.seek(offset);

  long contentLength = in.getVisibleLength() - offset;
  if (length >= 0) {
    contentLength = Math.min(contentLength, length);
  }
  final InputStream data;
  if (contentLength >= 0) {
    headers.set(CONTENT_LENGTH, contentLength);
    data = new LimitInputStream(in, contentLength);
  } else {
    data = in;
  }

  ctx.write(response);
  ctx.writeAndFlush(new ChunkedStream(data) {
    @Override
    public void close() throws Exception {
      super.close();
      dfsclient.close();
    }
  }).addListener(ChannelFutureListener.CLOSE);
}
 

public static ExtendedBlock getFirstBlock(FileSystem fs, Path path) throws IOException {
  HdfsDataInputStream in = (HdfsDataInputStream) fs.open(path);
  try {
    in.readByte();
    return in.getCurrentBlock();
  } finally {
    in.close();
  }
}
 

private boolean checkUnsupportedMethod(FileSystem fs, Path file,
                                         byte[] expected, int readOffset) throws IOException {
  HdfsDataInputStream stm = (HdfsDataInputStream)fs.open(file);
  ByteBuffer actual = ByteBuffer.allocateDirect(expected.length - readOffset);
  IOUtils.skipFully(stm, readOffset);
  try {
    stm.read(actual);
  } catch(UnsupportedOperationException unex) {
    return true;
  }
  return false;
}
 

/**
 * Read in-memory block with Short Circuit Read
 * Note: the test uses faked RAM_DISK from physical disk.
 */
@Test
public void testRamDiskShortCircuitRead()
  throws IOException, InterruptedException {
  startUpCluster(REPL_FACTOR,
    new StorageType[]{RAM_DISK, DEFAULT},
    2 * BLOCK_SIZE - 1, true);  // 1 replica + delta, SCR read
  final String METHOD_NAME = GenericTestUtils.getMethodName();
  final int SEED = 0xFADED;
  Path path = new Path("/" + METHOD_NAME + ".dat");

  makeRandomTestFile(path, BLOCK_SIZE, true, SEED);
  ensureFileReplicasOnStorageType(path, RAM_DISK);

  // Sleep for a short time to allow the lazy writer thread to do its job
  Thread.sleep(3 * LAZY_WRITER_INTERVAL_SEC * 1000);

  //assertThat(verifyReadRandomFile(path, BLOCK_SIZE, SEED), is(true));
  FSDataInputStream fis = fs.open(path);

  // Verify SCR read counters
  try {
    fis = fs.open(path);
    byte[] buf = new byte[BUFFER_LENGTH];
    fis.read(0, buf, 0, BUFFER_LENGTH);
    HdfsDataInputStream dfsis = (HdfsDataInputStream) fis;
    Assert.assertEquals(BUFFER_LENGTH,
      dfsis.getReadStatistics().getTotalBytesRead());
    Assert.assertEquals(BUFFER_LENGTH,
      dfsis.getReadStatistics().getTotalShortCircuitBytesRead());
  } finally {
    fis.close();
    fis = null;
  }
}
 

/** Check the file content, reading as user {@code readingUser} */
static void checkFileContentDirect(URI uri, Path name, byte[] expected,
    int readOffset, String readingUser, Configuration conf,
    boolean legacyShortCircuitFails)
    throws IOException, InterruptedException {
  // Ensure short circuit is enabled
  DistributedFileSystem fs = getFileSystem(readingUser, uri, conf);
  ClientContext clientContext = ClientContext.getFromConf(conf);
  if (legacyShortCircuitFails) {
    assertTrue(clientContext.getDisableLegacyBlockReaderLocal());
  }
  
  HdfsDataInputStream stm = (HdfsDataInputStream)fs.open(name);

  ByteBuffer actual = ByteBuffer.allocateDirect(expected.length - readOffset);

  IOUtils.skipFully(stm, readOffset);

  actual.limit(3);

  //Read a small number of bytes first.
  int nread = stm.read(actual);
  actual.limit(nread + 2);
  nread += stm.read(actual);

  // Read across chunk boundary
  actual.limit(Math.min(actual.capacity(), nread + 517));
  nread += stm.read(actual);
  checkData(arrayFromByteBuffer(actual), readOffset, expected, nread,
      "A few bytes");
  //Now read rest of it
  actual.limit(actual.capacity());
  while (actual.hasRemaining()) {
    int nbytes = stm.read(actual);

    if (nbytes < 0) {
      throw new EOFException("End of file reached before reading fully.");
    }
    nread += nbytes;
  }
  checkData(arrayFromByteBuffer(actual), readOffset, expected, "Read 3");
  if (legacyShortCircuitFails) {
    assertTrue(clientContext.getDisableLegacyBlockReaderLocal());
  }
  stm.close();
}
 

public static List<LocatedBlock> getAllBlocks(FSDataInputStream in)
    throws IOException {
  return ((HdfsDataInputStream) in).getAllBlocks();
}
 

public static List<LocatedBlock> getAllBlocks(FileSystem fs, Path path)
    throws IOException {
  HdfsDataInputStream in = (HdfsDataInputStream) fs.open(path);
  return in.getAllBlocks();
}
 

/**
 * Eviction of lazy persisted blocks with Short Circuit Read handle open
 * Note: the test uses faked RAM_DISK from physical disk.
 * @throws IOException
 * @throws InterruptedException
 */
@Test
public void testRamDiskEvictionWithShortCircuitReadHandle()
  throws IOException, InterruptedException {
  startUpCluster(REPL_FACTOR, new StorageType[] { RAM_DISK, DEFAULT },
    (6 * BLOCK_SIZE -1), true);  // 5 replica + delta, SCR.
  final String METHOD_NAME = GenericTestUtils.getMethodName();
  Path path1 = new Path("/" + METHOD_NAME + ".01.dat");
  Path path2 = new Path("/" + METHOD_NAME + ".02.dat");
  final int SEED = 0xFADED;

  makeRandomTestFile(path1, BLOCK_SIZE, true, SEED);
  ensureFileReplicasOnStorageType(path1, RAM_DISK);

  // Sleep for a short time to allow the lazy writer thread to do its job.
  // However the block replica should not be evicted from RAM_DISK yet.
  Thread.sleep(3 * LAZY_WRITER_INTERVAL_SEC * 1000);

  // No eviction should happen as the free ratio is below the threshold
  FSDataInputStream fis = fs.open(path1);
  try {
    // Keep and open read handle to path1 while creating path2
    byte[] buf = new byte[BUFFER_LENGTH];
    fis.read(0, buf, 0, BUFFER_LENGTH);

    // Create the 2nd file that will trigger RAM_DISK eviction.
    makeTestFile(path2, BLOCK_SIZE * 2, true);
    ensureFileReplicasOnStorageType(path2, RAM_DISK);

    // Ensure path1 is still readable from the open SCR handle.
    fis.read(fis.getPos(), buf, 0, BUFFER_LENGTH);
    HdfsDataInputStream dfsis = (HdfsDataInputStream) fis;
    Assert.assertEquals(2 * BUFFER_LENGTH,
      dfsis.getReadStatistics().getTotalBytesRead());
    Assert.assertEquals(2 * BUFFER_LENGTH,
      dfsis.getReadStatistics().getTotalShortCircuitBytesRead());
  } finally {
    IOUtils.closeQuietly(fis);
  }

  // After the open handle is closed, path1 should be evicted to DISK.
  triggerBlockReport();
  ensureFileReplicasOnStorageType(path1, DEFAULT);
}
 

/** Check the file content, reading as user {@code readingUser} */
static void checkFileContentDirect(URI uri, Path name, byte[] expected,
    int readOffset, String readingUser, Configuration conf,
    boolean legacyShortCircuitFails)
    throws IOException, InterruptedException {
  // Ensure short circuit is enabled
  DistributedFileSystem fs = getFileSystem(readingUser, uri, conf);
  ClientContext clientContext = ClientContext.getFromConf(conf);
  if (legacyShortCircuitFails) {
    assertTrue(clientContext.getDisableLegacyBlockReaderLocal());
  }
  
  HdfsDataInputStream stm = (HdfsDataInputStream)fs.open(name);

  ByteBuffer actual = ByteBuffer.allocateDirect(expected.length - readOffset);

  IOUtils.skipFully(stm, readOffset);

  actual.limit(3);

  //Read a small number of bytes first.
  int nread = stm.read(actual);
  actual.limit(nread + 2);
  nread += stm.read(actual);

  // Read across chunk boundary
  actual.limit(Math.min(actual.capacity(), nread + 517));
  nread += stm.read(actual);
  checkData(arrayFromByteBuffer(actual), readOffset, expected, nread,
      "A few bytes");
  //Now read rest of it
  actual.limit(actual.capacity());
  while (actual.hasRemaining()) {
    int nbytes = stm.read(actual);

    if (nbytes < 0) {
      throw new EOFException("End of file reached before reading fully.");
    }
    nread += nbytes;
  }
  checkData(arrayFromByteBuffer(actual), readOffset, expected, "Read 3");
  if (legacyShortCircuitFails) {
    assertTrue(clientContext.getDisableLegacyBlockReaderLocal());
  }
  stm.close();
}
 

public static List<LocatedBlock> getAllBlocks(FileSystem fs, Path path)
    throws IOException {
  HdfsDataInputStream in = (HdfsDataInputStream) fs.open(path);
  return in.getAllBlocks();
}
 

public static List<LocatedBlock> getAllBlocks(FSDataInputStream in)
    throws IOException {
  return ((HdfsDataInputStream) in).getAllBlocks();
}
 

/**
 * Eviction of lazy persisted blocks with Short Circuit Read handle open
 * Note: the test uses faked RAM_DISK from physical disk.
 * @throws IOException
 * @throws InterruptedException
 */
@Test
public void testRamDiskEvictionWithShortCircuitReadHandle()
  throws IOException, InterruptedException {
  startUpCluster(REPL_FACTOR, new StorageType[] { RAM_DISK, DEFAULT },
    (6 * BLOCK_SIZE -1), true);  // 5 replica + delta, SCR.
  final String METHOD_NAME = GenericTestUtils.getMethodName();
  Path path1 = new Path("/" + METHOD_NAME + ".01.dat");
  Path path2 = new Path("/" + METHOD_NAME + ".02.dat");
  final int SEED = 0xFADED;

  makeRandomTestFile(path1, BLOCK_SIZE, true, SEED);
  ensureFileReplicasOnStorageType(path1, RAM_DISK);

  // Sleep for a short time to allow the lazy writer thread to do its job.
  // However the block replica should not be evicted from RAM_DISK yet.
  Thread.sleep(3 * LAZY_WRITER_INTERVAL_SEC * 1000);

  // No eviction should happen as the free ratio is below the threshold
  FSDataInputStream fis = fs.open(path1);
  try {
    // Keep and open read handle to path1 while creating path2
    byte[] buf = new byte[BUFFER_LENGTH];
    fis.read(0, buf, 0, BUFFER_LENGTH);

    // Create the 2nd file that will trigger RAM_DISK eviction.
    makeTestFile(path2, BLOCK_SIZE * 2, true);
    ensureFileReplicasOnStorageType(path2, RAM_DISK);

    // Ensure path1 is still readable from the open SCR handle.
    fis.read(fis.getPos(), buf, 0, BUFFER_LENGTH);
    HdfsDataInputStream dfsis = (HdfsDataInputStream) fis;
    Assert.assertEquals(2 * BUFFER_LENGTH,
      dfsis.getReadStatistics().getTotalBytesRead());
    Assert.assertEquals(2 * BUFFER_LENGTH,
      dfsis.getReadStatistics().getTotalShortCircuitBytesRead());
  } finally {
    IOUtils.closeQuietly(fis);
  }

  // After the open handle is closed, path1 should be evicted to DISK.
  triggerBlockReport();
  ensureFileReplicasOnStorageType(path1, DEFAULT);
}
 

/**
 * Verify that the number of replicas are as expected for each block in
 * the given file.
 * For blocks with a decommissioned node, verify that their replication
 * is 1 more than what is specified.
 * For blocks without decommissioned nodes, verify their replication is
 * equal to what is specified.
 * 
 * @param downnode - if null, there is no decommissioned node for this file.
 * @return - null if no failure found, else an error message string.
 */
private static String checkFile(FileSystem fileSys, Path name, int repl,
  String downnode, int numDatanodes) throws IOException {
  boolean isNodeDown = (downnode != null);
  // need a raw stream
  assertTrue("Not HDFS:"+fileSys.getUri(),
      fileSys instanceof DistributedFileSystem);
  HdfsDataInputStream dis = (HdfsDataInputStream)
      fileSys.open(name);
  Collection<LocatedBlock> dinfo = dis.getAllBlocks();
  for (LocatedBlock blk : dinfo) { // for each block
    int hasdown = 0;
    DatanodeInfo[] nodes = blk.getLocations();
    for (int j = 0; j < nodes.length; j++) { // for each replica
      if (isNodeDown && nodes[j].getXferAddr().equals(downnode)) {
        hasdown++;
        //Downnode must actually be decommissioned
        if (!nodes[j].isDecommissioned()) {
          return "For block " + blk.getBlock() + " replica on " +
            nodes[j] + " is given as downnode, " +
            "but is not decommissioned";
        }
        //Decommissioned node (if any) should only be last node in list.
        if (j != nodes.length - 1) {
          return "For block " + blk.getBlock() + " decommissioned node "
            + nodes[j] + " was not last node in list: "
            + (j + 1) + " of " + nodes.length;
        }
        LOG.info("Block " + blk.getBlock() + " replica on " +
          nodes[j] + " is decommissioned.");
      } else {
        //Non-downnodes must not be decommissioned
        if (nodes[j].isDecommissioned()) {
          return "For block " + blk.getBlock() + " replica on " +
            nodes[j] + " is unexpectedly decommissioned";
        }
      }
    }

    LOG.info("Block " + blk.getBlock() + " has " + hasdown
      + " decommissioned replica.");
    if(Math.min(numDatanodes, repl+hasdown) != nodes.length) {
      return "Wrong number of replicas for block " + blk.getBlock() +
        ": " + nodes.length + ", expected " +
        Math.min(numDatanodes, repl+hasdown);
    }
  }
  return null;
}
 

@BeforeClass
public static void setUpClass() throws IOException, NoSuchFieldException, IllegalAccessException, ClassNotFoundException, NoSuchMethodException, InvocationTargetException, URISyntaxException, InstantiationException {
    classLoader = new ByteArrayClassLoader.ChildFirst(FileSystemTracerTest.class.getClassLoader(),
        ClassFileExtraction.of(
            Tracer.class,
            MethodTracer.class,
            FileSystemTracer.class,
            FileSystemTracer.WriteTracer.class,
            FileSystemTracer.ReadTracer.class,
            FileSystemTracer.AddBlockTracer.class,
            FileSystemTracer.ListStatusTracer.class,
            FileSystemTracer.GetContentSummaryTracer.class,
            FileSystemTracer.RenameTracer.class,
            FileSystemTracer.DeleteTracer.class,
            DFSClient.class,
            DFSClient.Conf.class,
            ClientContext.class,
            DistributedFileSystem.class,
            DomainSocketFactory.class,
            DFSInputStream.class,
            DFSOutputStream.class,
            HdfsDataInputStream.class,
            HdfsDataOutputStream.class,
            ClientNamenodeProtocolTranslatorPB.class,
            Class.forName("org.apache.hadoop.hdfs.DFSOpsCountStatistics"),
            Class.forName("org.apache.hadoop.hdfs.BlockReaderLocal"),
            Class.forName(DFSOutputStream.class.getName() + "$Packet"),
            Class.forName(DFSOutputStream.class.getName() + "$DataStreamer"),
            Class.forName(DFSOutputStream.class.getName() + "$DataStreamer$1"),
            Class.forName(DFSOutputStream.class.getName() + "$DataStreamer$2"),
            Class.forName(DFSOutputStream.class.getName() + "$DataStreamer$ResponseProcessor"),
            Class.forName(DistributedFileSystem.class.getName() + "$1"),
            Class.forName(DistributedFileSystem.class.getName() + "$4"),
            Class.forName(DistributedFileSystem.class.getName() + "$7"),
            Class.forName(DistributedFileSystem.class.getName() + "$13"),
            Class.forName(DistributedFileSystem.class.getName() + "$14"),
            Class.forName(DistributedFileSystem.class.getName() + "$16"),
            Class.forName(DistributedFileSystem.class.getName() + "$19"),
            Class.forName("org.apache.hadoop.hdfs.LeaseRenewer"),
            Class.forName("org.apache.hadoop.hdfs.LeaseRenewer$1"),
            Class.forName("org.apache.hadoop.hdfs.LeaseRenewer$Factory"),
            Class.forName("org.apache.hadoop.hdfs.LeaseRenewer$Factory$Key"),
            Class.forName("org.apache.hadoop.fs.Hdfs"),
            HdfsDataOutputStream.class
        ),
        ByteArrayClassLoader.PersistenceHandler.MANIFEST);

    argument = ArgumentCaptor.forClass(DataAccessEventProtos.FsEvent.class);

    eventHandler = mock(BiConsumer.class);

    ReflectionHelper.setField(null, classLoader.loadClass(FileSystemTracer.class.getName()), "eventHandler", eventHandler);
    ReflectionHelper.setField(conf, Configuration.class, "classLoader", classLoader);
    assertThat(ByteBuddyAgent.install(), instanceOf(Instrumentation.class));

    new FileSystemTracer.WriteTracer().installOnByteBuddyAgent();
    new FileSystemTracer.ReadTracer().installOnByteBuddyAgent();
    new FileSystemTracer.ListStatusTracer().installOnByteBuddyAgent();
    new FileSystemTracer.GetContentSummaryTracer().installOnByteBuddyAgent();
    new FileSystemTracer.DeleteTracer().installOnByteBuddyAgent();
    new FileSystemTracer.RenameTracer().installOnByteBuddyAgent();

    // MiniDfsCluster
    File baseDir = new File("./target/hdfs/test").getAbsoluteFile();
    FileUtil.fullyDelete(baseDir);
    conf.set("fs.hdfs.impl", "org.apache.hadoop.hdfs.DistributedFileSystem");
    conf.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, baseDir.getAbsolutePath());
    MiniDFSCluster.Builder builder = new MiniDFSCluster.Builder(conf)
        .simulatedCapacities(new long[] {10240000L});
    hdfsCluster = builder.build();
    hdfsURI = "hdfs://localhost:" + hdfsCluster.getNameNodePort();

    initDFS();
}
 

/**
 * Verify that the number of replicas are as expected for each block in
 * the given file.
 * For blocks with a decommissioned node, verify that their replication
 * is 1 more than what is specified.
 * For blocks without decommissioned nodes, verify their replication is
 * equal to what is specified.
 * 
 * @param downnode - if null, there is no decommissioned node for this file.
 * @return - null if no failure found, else an error message string.
 */
private static String checkFile(FileSystem fileSys, Path name, int repl,
  String downnode, int numDatanodes) throws IOException {
  boolean isNodeDown = (downnode != null);
  // need a raw stream
  assertTrue("Not HDFS:"+fileSys.getUri(),
      fileSys instanceof DistributedFileSystem);
  HdfsDataInputStream dis = (HdfsDataInputStream)
      fileSys.open(name);
  Collection<LocatedBlock> dinfo = dis.getAllBlocks();
  for (LocatedBlock blk : dinfo) { // for each block
    int hasdown = 0;
    DatanodeInfo[] nodes = blk.getLocations();
    for (int j = 0; j < nodes.length; j++) { // for each replica
      if (isNodeDown && nodes[j].getXferAddr().equals(downnode)) {
        hasdown++;
        //Downnode must actually be decommissioned
        if (!nodes[j].isDecommissioned()) {
          return "For block " + blk.getBlock() + " replica on " +
            nodes[j] + " is given as downnode, " +
            "but is not decommissioned";
        }
        //Decommissioned node (if any) should only be last node in list.
        if (j != nodes.length - 1) {
          return "For block " + blk.getBlock() + " decommissioned node "
            + nodes[j] + " was not last node in list: "
            + (j + 1) + " of " + nodes.length;
        }
        LOG.info("Block " + blk.getBlock() + " replica on " +
          nodes[j] + " is decommissioned.");
      } else {
        //Non-downnodes must not be decommissioned
        if (nodes[j].isDecommissioned()) {
          return "For block " + blk.getBlock() + " replica on " +
            nodes[j] + " is unexpectedly decommissioned";
        }
      }
    }

    LOG.info("Block " + blk.getBlock() + " has " + hasdown
      + " decommissioned replica.");
    if(Math.min(numDatanodes, repl+hasdown) != nodes.length) {
      return "Wrong number of replicas for block " + blk.getBlock() +
        ": " + nodes.length + ", expected " +
        Math.min(numDatanodes, repl+hasdown);
    }
  }
  return null;
}