Java Code Examples for org.apache.hadoop.hdfs.protocol.ExtendedBlock#getLocalBlock()

The following examples show how to use org.apache.hadoop.hdfs.protocol.ExtendedBlock#getLocalBlock() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: SimulatedFSDataset.java    From hadoop with Apache License 2.0 6 votes vote down vote up 
@Override // FsDatasetSpi
public synchronized ReplicaHandler createTemporary(
    StorageType storageType, ExtendedBlock b) throws IOException {
  if (isValidBlock(b)) {
        throw new ReplicaAlreadyExistsException("Block " + b + 
            " is valid, and cannot be written to.");
    }
  if (isValidRbw(b)) {
      throw new ReplicaAlreadyExistsException("Block " + b + 
          " is being written, and cannot be written to.");
  }
  final Map<Block, BInfo> map = getMap(b.getBlockPoolId());
  BInfo binfo = new BInfo(b.getBlockPoolId(), b.getLocalBlock(), true);
  map.put(binfo.theBlock, binfo);
  return new ReplicaHandler(binfo, null);
}
Example 2
Source File: SimulatedFSDataset.java    From big-c with Apache License 2.0 6 votes vote down vote up 
@Override // FsDatasetSpi
public synchronized ReplicaHandler createTemporary(
    StorageType storageType, ExtendedBlock b) throws IOException {
  if (isValidBlock(b)) {
        throw new ReplicaAlreadyExistsException("Block " + b + 
            " is valid, and cannot be written to.");
    }
  if (isValidRbw(b)) {
      throw new ReplicaAlreadyExistsException("Block " + b + 
          " is being written, and cannot be written to.");
  }
  final Map<Block, BInfo> map = getMap(b.getBlockPoolId());
  BInfo binfo = new BInfo(b.getBlockPoolId(), b.getLocalBlock(), true);
  map.put(binfo.theBlock, binfo);
  return new ReplicaHandler(binfo, null);
}
Example 3
Source File: BPOfferService.java    From hadoop with Apache License 2.0 5 votes vote down vote up 
void notifyNamenodeReceivedBlock(
    ExtendedBlock block, String delHint, String storageUuid) {
  checkBlock(block);
  ReceivedDeletedBlockInfo bInfo = new ReceivedDeletedBlockInfo(
      block.getLocalBlock(),
      ReceivedDeletedBlockInfo.BlockStatus.RECEIVED_BLOCK,
      delHint);

  for (BPServiceActor actor : bpServices) {
    actor.notifyNamenodeBlock(bInfo, storageUuid, true);
  }
}
Example 4
Source File: BPOfferService.java    From hadoop with Apache License 2.0 5 votes vote down vote up 
void notifyNamenodeDeletedBlock(ExtendedBlock block, String storageUuid) {
  checkBlock(block);
  ReceivedDeletedBlockInfo bInfo = new ReceivedDeletedBlockInfo(
     block.getLocalBlock(), BlockStatus.DELETED_BLOCK, null);
  
  for (BPServiceActor actor : bpServices) {
    actor.notifyNamenodeDeletedBlock(bInfo, storageUuid);
  }
}
Example 5
Source File: BPOfferService.java    From hadoop with Apache License 2.0 5 votes vote down vote up 
void notifyNamenodeReceivingBlock(ExtendedBlock block, String storageUuid) {
  checkBlock(block);
  ReceivedDeletedBlockInfo bInfo = new ReceivedDeletedBlockInfo(
     block.getLocalBlock(), BlockStatus.RECEIVING_BLOCK, null);
  
  for (BPServiceActor actor : bpServices) {
    actor.notifyNamenodeBlock(bInfo, storageUuid, false);
  }
}
Example 6
Source File: BPOfferService.java    From big-c with Apache License 2.0 5 votes vote down vote up 
void notifyNamenodeReceivedBlock(
    ExtendedBlock block, String delHint, String storageUuid) {
  checkBlock(block);
  ReceivedDeletedBlockInfo bInfo = new ReceivedDeletedBlockInfo(
      block.getLocalBlock(),
      ReceivedDeletedBlockInfo.BlockStatus.RECEIVED_BLOCK,
      delHint);

  for (BPServiceActor actor : bpServices) {
    actor.notifyNamenodeBlock(bInfo, storageUuid, true);
  }
}
Example 7
Source File: BPOfferService.java    From big-c with Apache License 2.0 5 votes vote down vote up 
void notifyNamenodeDeletedBlock(ExtendedBlock block, String storageUuid) {
  checkBlock(block);
  ReceivedDeletedBlockInfo bInfo = new ReceivedDeletedBlockInfo(
     block.getLocalBlock(), BlockStatus.DELETED_BLOCK, null);
  
  for (BPServiceActor actor : bpServices) {
    actor.notifyNamenodeDeletedBlock(bInfo, storageUuid);
  }
}
Example 8
Source File: BPOfferService.java    From big-c with Apache License 2.0 5 votes vote down vote up 
void notifyNamenodeReceivingBlock(ExtendedBlock block, String storageUuid) {
  checkBlock(block);
  ReceivedDeletedBlockInfo bInfo = new ReceivedDeletedBlockInfo(
     block.getLocalBlock(), BlockStatus.RECEIVING_BLOCK, null);
  
  for (BPServiceActor actor : bpServices) {
    actor.notifyNamenodeBlock(bInfo, storageUuid, false);
  }
}
Example 9
Source File: TestRBWBlockInvalidation.java    From hadoop with Apache License 2.0 4 votes vote down vote up 
/**
 * Test when a block's replica is removed from RBW folder in one of the
 * datanode, namenode should ask to invalidate that corrupted block and
 * schedule replication for one more replica for that under replicated block.
 */
@Test(timeout=600000)
public void testBlockInvalidationWhenRBWReplicaMissedInDN()
    throws IOException, InterruptedException {
  // This test cannot pass on Windows due to file locking enforcement.  It will
  // reject the attempt to delete the block file from the RBW folder.
  assumeTrue(!Path.WINDOWS);

  Configuration conf = new HdfsConfiguration();
  conf.setInt(DFSConfigKeys.DFS_REPLICATION_KEY, 2);
  conf.setLong(DFSConfigKeys.DFS_BLOCKREPORT_INTERVAL_MSEC_KEY, 300);
  conf.setLong(DFSConfigKeys.DFS_DATANODE_DIRECTORYSCAN_INTERVAL_KEY, 1);
  conf.setLong(DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY, 1);
  MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(2)
      .build();
  FSDataOutputStream out = null;
  try {
    final FSNamesystem namesystem = cluster.getNamesystem();
    FileSystem fs = cluster.getFileSystem();
    Path testPath = new Path("/tmp/TestRBWBlockInvalidation", "foo1");
    out = fs.create(testPath, (short) 2);
    out.writeBytes("HDFS-3157: " + testPath);
    out.hsync();
    cluster.startDataNodes(conf, 1, true, null, null, null);
    String bpid = namesystem.getBlockPoolId();
    ExtendedBlock blk = DFSTestUtil.getFirstBlock(fs, testPath);
    Block block = blk.getLocalBlock();
    DataNode dn = cluster.getDataNodes().get(0);

    // Delete partial block and its meta information from the RBW folder
    // of first datanode.
    File blockFile = DataNodeTestUtils.getBlockFile(dn, bpid, block);
    File metaFile = DataNodeTestUtils.getMetaFile(dn, bpid, block);
    assertTrue("Could not delete the block file from the RBW folder",
        blockFile.delete());
    assertTrue("Could not delete the block meta file from the RBW folder",
        metaFile.delete());

    out.close();
    
    int liveReplicas = 0;
    while (true) {
      if ((liveReplicas = countReplicas(namesystem, blk).liveReplicas()) < 2) {
        // This confirms we have a corrupt replica
        LOG.info("Live Replicas after corruption: " + liveReplicas);
        break;
      }
      Thread.sleep(100);
    }
    assertEquals("There should be less than 2 replicas in the "
        + "liveReplicasMap", 1, liveReplicas);
    
    while (true) {
      if ((liveReplicas =
            countReplicas(namesystem, blk).liveReplicas()) > 1) {
        //Wait till the live replica count becomes equal to Replication Factor
        LOG.info("Live Replicas after Rereplication: " + liveReplicas);
        break;
      }
      Thread.sleep(100);
    }
    assertEquals("There should be two live replicas", 2, liveReplicas);

    while (true) {
      Thread.sleep(100);
      if (countReplicas(namesystem, blk).corruptReplicas() == 0) {
        LOG.info("Corrupt Replicas becomes 0");
        break;
      }
    }
  } finally {
    if (out != null) {
      out.close();
    }
    cluster.shutdown();
  }
}
Example 10
Source File: TestRBWBlockInvalidation.java    From big-c with Apache License 2.0 4 votes vote down vote up 
/**
 * Test when a block's replica is removed from RBW folder in one of the
 * datanode, namenode should ask to invalidate that corrupted block and
 * schedule replication for one more replica for that under replicated block.
 */
@Test(timeout=600000)
public void testBlockInvalidationWhenRBWReplicaMissedInDN()
    throws IOException, InterruptedException {
  // This test cannot pass on Windows due to file locking enforcement.  It will
  // reject the attempt to delete the block file from the RBW folder.
  assumeTrue(!Path.WINDOWS);

  Configuration conf = new HdfsConfiguration();
  conf.setInt(DFSConfigKeys.DFS_REPLICATION_KEY, 2);
  conf.setLong(DFSConfigKeys.DFS_BLOCKREPORT_INTERVAL_MSEC_KEY, 300);
  conf.setLong(DFSConfigKeys.DFS_DATANODE_DIRECTORYSCAN_INTERVAL_KEY, 1);
  conf.setLong(DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY, 1);
  MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(2)
      .build();
  FSDataOutputStream out = null;
  try {
    final FSNamesystem namesystem = cluster.getNamesystem();
    FileSystem fs = cluster.getFileSystem();
    Path testPath = new Path("/tmp/TestRBWBlockInvalidation", "foo1");
    out = fs.create(testPath, (short) 2);
    out.writeBytes("HDFS-3157: " + testPath);
    out.hsync();
    cluster.startDataNodes(conf, 1, true, null, null, null);
    String bpid = namesystem.getBlockPoolId();
    ExtendedBlock blk = DFSTestUtil.getFirstBlock(fs, testPath);
    Block block = blk.getLocalBlock();
    DataNode dn = cluster.getDataNodes().get(0);

    // Delete partial block and its meta information from the RBW folder
    // of first datanode.
    File blockFile = DataNodeTestUtils.getBlockFile(dn, bpid, block);
    File metaFile = DataNodeTestUtils.getMetaFile(dn, bpid, block);
    assertTrue("Could not delete the block file from the RBW folder",
        blockFile.delete());
    assertTrue("Could not delete the block meta file from the RBW folder",
        metaFile.delete());

    out.close();
    
    int liveReplicas = 0;
    while (true) {
      if ((liveReplicas = countReplicas(namesystem, blk).liveReplicas()) < 2) {
        // This confirms we have a corrupt replica
        LOG.info("Live Replicas after corruption: " + liveReplicas);
        break;
      }
      Thread.sleep(100);
    }
    assertEquals("There should be less than 2 replicas in the "
        + "liveReplicasMap", 1, liveReplicas);
    
    while (true) {
      if ((liveReplicas =
            countReplicas(namesystem, blk).liveReplicas()) > 1) {
        //Wait till the live replica count becomes equal to Replication Factor
        LOG.info("Live Replicas after Rereplication: " + liveReplicas);
        break;
      }
      Thread.sleep(100);
    }
    assertEquals("There should be two live replicas", 2, liveReplicas);

    while (true) {
      Thread.sleep(100);
      if (countReplicas(namesystem, blk).corruptReplicas() == 0) {
        LOG.info("Corrupt Replicas becomes 0");
        break;
      }
    }
  } finally {
    if (out != null) {
      out.close();
    }
    cluster.shutdown();
  }
}