Java Code Examples for org.apache.hadoop.hdfs.DFSTestUtil#getLocalDatanodeDescriptor()

The following examples show how to use org.apache.hadoop.hdfs.DFSTestUtil#getLocalDatanodeDescriptor() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: TestDatanodeDescriptor.java    From hadoop with Apache License 2.0 6 votes vote down vote up 
/**
 * Test that getInvalidateBlocks observes the maxlimit.
 */
@Test
public void testGetInvalidateBlocks() throws Exception {
  final int MAX_BLOCKS = 10;
  final int REMAINING_BLOCKS = 2;
  final int MAX_LIMIT = MAX_BLOCKS - REMAINING_BLOCKS;
  
  DatanodeDescriptor dd = DFSTestUtil.getLocalDatanodeDescriptor();
  ArrayList<Block> blockList = new ArrayList<Block>(MAX_BLOCKS);
  for (int i=0; i<MAX_BLOCKS; i++) {
    blockList.add(new Block(i, 0, GenerationStamp.LAST_RESERVED_STAMP));
  }
  dd.addBlocksToBeInvalidated(blockList);
  Block[] bc = dd.getInvalidateBlocks(MAX_LIMIT);
  assertEquals(bc.length, MAX_LIMIT);
  bc = dd.getInvalidateBlocks(MAX_LIMIT);
  assertEquals(bc.length, REMAINING_BLOCKS);
}
Example 2
Source File: TestPendingDataNodeMessages.java    From hadoop with Apache License 2.0 6 votes vote down vote up 
@Test
public void testQueues() {
  DatanodeDescriptor fakeDN = DFSTestUtil.getLocalDatanodeDescriptor();
  DatanodeStorage storage = new DatanodeStorage("STORAGE_ID");
  DatanodeStorageInfo storageInfo = new DatanodeStorageInfo(fakeDN, storage);
  msgs.enqueueReportedBlock(storageInfo, block1Gs1, ReplicaState.FINALIZED);
  msgs.enqueueReportedBlock(storageInfo, block1Gs2, ReplicaState.FINALIZED);

  assertEquals(2, msgs.count());
  
  // Nothing queued yet for block 2
  assertNull(msgs.takeBlockQueue(block2Gs1));
  assertEquals(2, msgs.count());
  
  Queue<ReportedBlockInfo> q =
    msgs.takeBlockQueue(block1Gs2DifferentInstance);
  assertEquals(
      "ReportedBlockInfo [block=blk_1_1, dn=127.0.0.1:50010, reportedState=FINALIZED]," +
      "ReportedBlockInfo [block=blk_1_2, dn=127.0.0.1:50010, reportedState=FINALIZED]",
      Joiner.on(",").join(q));
  assertEquals(0, msgs.count());
  
  // Should be null if we pull again
  assertNull(msgs.takeBlockQueue(block1Gs1));
  assertEquals(0, msgs.count());
}
Example 3
Source File: TestDatanodeDescriptor.java    From big-c with Apache License 2.0 6 votes vote down vote up 
/**
 * Test that getInvalidateBlocks observes the maxlimit.
 */
@Test
public void testGetInvalidateBlocks() throws Exception {
  final int MAX_BLOCKS = 10;
  final int REMAINING_BLOCKS = 2;
  final int MAX_LIMIT = MAX_BLOCKS - REMAINING_BLOCKS;
  
  DatanodeDescriptor dd = DFSTestUtil.getLocalDatanodeDescriptor();
  ArrayList<Block> blockList = new ArrayList<Block>(MAX_BLOCKS);
  for (int i=0; i<MAX_BLOCKS; i++) {
    blockList.add(new Block(i, 0, GenerationStamp.LAST_RESERVED_STAMP));
  }
  dd.addBlocksToBeInvalidated(blockList);
  Block[] bc = dd.getInvalidateBlocks(MAX_LIMIT);
  assertEquals(bc.length, MAX_LIMIT);
  bc = dd.getInvalidateBlocks(MAX_LIMIT);
  assertEquals(bc.length, REMAINING_BLOCKS);
}
Example 4
Source File: TestPendingDataNodeMessages.java    From big-c with Apache License 2.0 6 votes vote down vote up 
@Test
public void testQueues() {
  DatanodeDescriptor fakeDN = DFSTestUtil.getLocalDatanodeDescriptor();
  DatanodeStorage storage = new DatanodeStorage("STORAGE_ID");
  DatanodeStorageInfo storageInfo = new DatanodeStorageInfo(fakeDN, storage);
  msgs.enqueueReportedBlock(storageInfo, block1Gs1, ReplicaState.FINALIZED);
  msgs.enqueueReportedBlock(storageInfo, block1Gs2, ReplicaState.FINALIZED);

  assertEquals(2, msgs.count());
  
  // Nothing queued yet for block 2
  assertNull(msgs.takeBlockQueue(block2Gs1));
  assertEquals(2, msgs.count());
  
  Queue<ReportedBlockInfo> q =
    msgs.takeBlockQueue(block1Gs2DifferentInstance);
  assertEquals(
      "ReportedBlockInfo [block=blk_1_1, dn=127.0.0.1:50010, reportedState=FINALIZED]," +
      "ReportedBlockInfo [block=blk_1_2, dn=127.0.0.1:50010, reportedState=FINALIZED]",
      Joiner.on(",").join(q));
  assertEquals(0, msgs.count());
  
  // Should be null if we pull again
  assertNull(msgs.takeBlockQueue(block1Gs1));
  assertEquals(0, msgs.count());
}
Example 5
Source File: TestBlockManager.java    From hadoop with Apache License 2.0 5 votes vote down vote up 
@Test
public void testUseDelHint() {
  DatanodeStorageInfo delHint = new DatanodeStorageInfo(
      DFSTestUtil.getLocalDatanodeDescriptor(), new DatanodeStorage("id"));
  List<DatanodeStorageInfo> moreThan1Racks = Arrays.asList(delHint);
  List<StorageType> excessTypes = new ArrayList<StorageType>();

  excessTypes.add(StorageType.DEFAULT);
  Assert.assertTrue(BlockManager.useDelHint(true, delHint, null,
      moreThan1Racks, excessTypes));
  excessTypes.remove(0);
  excessTypes.add(StorageType.SSD);
  Assert.assertFalse(BlockManager.useDelHint(true, delHint, null,
      moreThan1Racks, excessTypes));
}
Example 6
Source File: TestBlockManager.java    From big-c with Apache License 2.0 5 votes vote down vote up 
@Test
public void testUseDelHint() {
  DatanodeStorageInfo delHint = new DatanodeStorageInfo(
      DFSTestUtil.getLocalDatanodeDescriptor(), new DatanodeStorage("id"));
  List<DatanodeStorageInfo> moreThan1Racks = Arrays.asList(delHint);
  List<StorageType> excessTypes = new ArrayList<StorageType>();

  excessTypes.add(StorageType.DEFAULT);
  Assert.assertTrue(BlockManager.useDelHint(true, delHint, null,
      moreThan1Racks, excessTypes));
  excessTypes.remove(0);
  excessTypes.add(StorageType.SSD);
  Assert.assertFalse(BlockManager.useDelHint(true, delHint, null,
      moreThan1Racks, excessTypes));
}
Example 7
Source File: TestCorruptReplicaInfo.java    From hadoop with Apache License 2.0 4 votes vote down vote up 
@Test
public void testCorruptReplicaInfo() throws IOException, 
                                     InterruptedException {
  
    CorruptReplicasMap crm = new CorruptReplicasMap();
    
    // Make sure initial values are returned correctly
    assertEquals("Number of corrupt blocks must initially be 0", 0, crm.size());
    assertNull("Param n cannot be less than 0", crm.getCorruptReplicaBlockIds(-1, null));
    assertNull("Param n cannot be greater than 100", crm.getCorruptReplicaBlockIds(101, null));
    long[] l = crm.getCorruptReplicaBlockIds(0, null);
    assertNotNull("n = 0 must return non-null", l);
    assertEquals("n = 0 must return an empty list", 0, l.length);

    // create a list of block_ids. A list is used to allow easy validation of the
    // output of getCorruptReplicaBlockIds
    int NUM_BLOCK_IDS = 140;
    List<Long> block_ids = new LinkedList<Long>();
    for (int i=0;i<NUM_BLOCK_IDS;i++) {
      block_ids.add((long)i);
    }
    
    DatanodeDescriptor dn1 = DFSTestUtil.getLocalDatanodeDescriptor();
    DatanodeDescriptor dn2 = DFSTestUtil.getLocalDatanodeDescriptor();
    
    addToCorruptReplicasMap(crm, getBlock(0), dn1);
    assertEquals("Number of corrupt blocks not returning correctly",
                 1, crm.size());
    addToCorruptReplicasMap(crm, getBlock(1), dn1);
    assertEquals("Number of corrupt blocks not returning correctly",
                 2, crm.size());
    
    addToCorruptReplicasMap(crm, getBlock(1), dn2);
    assertEquals("Number of corrupt blocks not returning correctly",
                 2, crm.size());
    
    crm.removeFromCorruptReplicasMap(getBlock(1));
    assertEquals("Number of corrupt blocks not returning correctly",
                 1, crm.size());
    
    crm.removeFromCorruptReplicasMap(getBlock(0));
    assertEquals("Number of corrupt blocks not returning correctly",
                 0, crm.size());
    
    for (Long block_id: block_ids) {
      addToCorruptReplicasMap(crm, getBlock(block_id), dn1);
    }
          
    assertEquals("Number of corrupt blocks not returning correctly",
                 NUM_BLOCK_IDS, crm.size());
    
    assertTrue("First five block ids not returned correctly ",
              Arrays.equals(new long[]{0,1,2,3,4},
                            crm.getCorruptReplicaBlockIds(5, null)));
                            
    LOG.info(crm.getCorruptReplicaBlockIds(10, 7L));
    LOG.info(block_ids.subList(7, 18));

    assertTrue("10 blocks after 7 not returned correctly ",
              Arrays.equals(new long[]{8,9,10,11,12,13,14,15,16,17},
                            crm.getCorruptReplicaBlockIds(10, 7L)));
    
}
Example 8
Source File: TestCorruptReplicaInfo.java    From big-c with Apache License 2.0 4 votes vote down vote up 
@Test
public void testCorruptReplicaInfo() throws IOException, 
                                     InterruptedException {
  
    CorruptReplicasMap crm = new CorruptReplicasMap();
    
    // Make sure initial values are returned correctly
    assertEquals("Number of corrupt blocks must initially be 0", 0, crm.size());
    assertNull("Param n cannot be less than 0", crm.getCorruptReplicaBlockIds(-1, null));
    assertNull("Param n cannot be greater than 100", crm.getCorruptReplicaBlockIds(101, null));
    long[] l = crm.getCorruptReplicaBlockIds(0, null);
    assertNotNull("n = 0 must return non-null", l);
    assertEquals("n = 0 must return an empty list", 0, l.length);

    // create a list of block_ids. A list is used to allow easy validation of the
    // output of getCorruptReplicaBlockIds
    int NUM_BLOCK_IDS = 140;
    List<Long> block_ids = new LinkedList<Long>();
    for (int i=0;i<NUM_BLOCK_IDS;i++) {
      block_ids.add((long)i);
    }
    
    DatanodeDescriptor dn1 = DFSTestUtil.getLocalDatanodeDescriptor();
    DatanodeDescriptor dn2 = DFSTestUtil.getLocalDatanodeDescriptor();
    
    addToCorruptReplicasMap(crm, getBlock(0), dn1);
    assertEquals("Number of corrupt blocks not returning correctly",
                 1, crm.size());
    addToCorruptReplicasMap(crm, getBlock(1), dn1);
    assertEquals("Number of corrupt blocks not returning correctly",
                 2, crm.size());
    
    addToCorruptReplicasMap(crm, getBlock(1), dn2);
    assertEquals("Number of corrupt blocks not returning correctly",
                 2, crm.size());
    
    crm.removeFromCorruptReplicasMap(getBlock(1));
    assertEquals("Number of corrupt blocks not returning correctly",
                 1, crm.size());
    
    crm.removeFromCorruptReplicasMap(getBlock(0));
    assertEquals("Number of corrupt blocks not returning correctly",
                 0, crm.size());
    
    for (Long block_id: block_ids) {
      addToCorruptReplicasMap(crm, getBlock(block_id), dn1);
    }
          
    assertEquals("Number of corrupt blocks not returning correctly",
                 NUM_BLOCK_IDS, crm.size());
    
    assertTrue("First five block ids not returned correctly ",
              Arrays.equals(new long[]{0,1,2,3,4},
                            crm.getCorruptReplicaBlockIds(5, null)));
                            
    LOG.info(crm.getCorruptReplicaBlockIds(10, 7L));
    LOG.info(block_ids.subList(7, 18));

    assertTrue("10 blocks after 7 not returned correctly ",
              Arrays.equals(new long[]{8,9,10,11,12,13,14,15,16,17},
                            crm.getCorruptReplicaBlockIds(10, 7L)));
    
}