org.apache.ratis.server.storage.FileInfo Java Examples

public SingleFileSnapshotInfo findLatestSnapshot() throws IOException {
  SingleFileSnapshotInfo latest = null;
  try (DirectoryStream<Path> stream =
           Files.newDirectoryStream(smDir.toPath())) {
    for (Path path : stream) {
      Matcher matcher = SNAPSHOT_REGEX.matcher(path.getFileName().toString());
      if (matcher.matches()) {
        final long endIndex = Long.parseLong(matcher.group(2));
        if (latest == null || endIndex > latest.getIndex()) {
          final long term = Long.parseLong(matcher.group(1));
          MD5Hash fileDigest = MD5FileUtil.readStoredMd5ForFile(path.toFile());
          final FileInfo fileInfo = new FileInfo(path, fileDigest);
          latest = new SingleFileSnapshotInfo(fileInfo, term, endIndex);
        }
      }
    }
  }
  return latest;
}
 

public SingleFileSnapshotInfo findLatestSnapshot() throws IOException {
  SingleFileSnapshotInfo latest = null;
  try (DirectoryStream<Path> stream =
           Files.newDirectoryStream(smDir.toPath())) {
    for (Path path : stream) {
      Matcher matcher = SNAPSHOT_REGEX.matcher(path.getFileName().toString());
      if (matcher.matches()) {
        final long endIndex = Long.parseLong(matcher.group(2));
        if (latest == null || endIndex > latest.getIndex()) {
          final long term = Long.parseLong(matcher.group(1));
          MD5Hash fileDigest = MD5FileUtil.readStoredMd5ForFile(path.toFile());
          final FileInfo fileInfo = new FileInfo(path, fileDigest);
          latest = new SingleFileSnapshotInfo(fileInfo, term, endIndex);
        }
      }
    }
  }
  return latest;
}
 

private FileChunkProto readFileChunk(FileInfo fileInfo,
    FileInputStream in, byte[] buf, int length, long offset, int chunkIndex)
    throws IOException {
  FileChunkProto.Builder builder = FileChunkProto.newBuilder()
      .setOffset(offset).setChunkIndex(chunkIndex);
  IOUtils.readFully(in, buf, 0, length);
  Path relativePath = server.getState().getStorage().getStorageDir()
      .relativizeToRoot(fileInfo.getPath());
  builder.setFilename(relativePath.toString());
  builder.setDone(offset + length == fileInfo.getFileSize());
  builder.setFileDigest(
      ByteString.copyFrom(fileInfo.getFileDigest().getDigest()));
  builder.setData(ByteString.copyFrom(buf, 0, length));
  return builder.build();
}
 

@Override
public void cleanupOldSnapshots(SnapshotRetentionPolicy snapshotRetentionPolicy) throws IOException {
  if (snapshotRetentionPolicy != null && snapshotRetentionPolicy.getNumSnapshotsRetained() > 0) {

    List<SingleFileSnapshotInfo> allSnapshotFiles = new ArrayList<>();
    try (DirectoryStream<Path> stream =
             Files.newDirectoryStream(smDir.toPath())) {
      for (Path path : stream) {
        Matcher matcher = SNAPSHOT_REGEX.matcher(path.getFileName().toString());
        if (matcher.matches()) {
          final long endIndex = Long.parseLong(matcher.group(2));
          final long term = Long.parseLong(matcher.group(1));
          final FileInfo fileInfo = new FileInfo(path, null); //We don't need FileDigest here.
          allSnapshotFiles.add(new SingleFileSnapshotInfo(fileInfo, term, endIndex));
        }
      }
    }

    if (allSnapshotFiles.size() > snapshotRetentionPolicy.getNumSnapshotsRetained()) {
      allSnapshotFiles.sort(new SnapshotFileComparator());
      List<File> snapshotFilesToBeCleaned = allSnapshotFiles.subList(
          snapshotRetentionPolicy.getNumSnapshotsRetained(), allSnapshotFiles.size()).stream()
          .map(singleFileSnapshotInfo -> singleFileSnapshotInfo.getFile().getPath().toFile())
          .collect(Collectors.toList());
      for (File snapshotFile : snapshotFilesToBeCleaned) {
        LOG.info("Deleting old snapshot at {}", snapshotFile.getAbsolutePath());
        FileUtils.deleteFileQuietly(snapshotFile);
      }
    }
  }
}
 

private FileChunkProto readFileChunk(FileInfo fileInfo,
    FileInputStream in, byte[] buf, int length, long offset, int chunkIndex)
    throws IOException {
  FileChunkProto.Builder builder = FileChunkProto.newBuilder()
      .setOffset(offset).setChunkIndex(chunkIndex);
  IOUtils.readFully(in, buf, 0, length);
  Path relativePath = server.getState().getStorage().getStorageDir()
      .relativizeToRoot(fileInfo.getPath());
  builder.setFilename(relativePath.toString());
  builder.setDone(offset + length == fileInfo.getFileSize());
  builder.setFileDigest(
      ByteString.copyFrom(fileInfo.getFileDigest().getDigest()));
  builder.setData(ByteString.copyFrom(buf, 0, length));
  return builder.build();
}
 

/** @return the file associated with the snapshot. */
public FileInfo getFile() {
  return getFiles().get(0);
}
 

/** @return the file associated with the snapshot. */
public FileInfo getFile() {
  return getFiles().get(0);
}
 

public SingleFileSnapshotInfo(FileInfo fileInfo, long term, long endIndex) {
  super(Arrays.asList(fileInfo), term, endIndex);
}
 

@Override
public List<FileInfo> getFiles() {
  return files;
}
 

public FileListSnapshotInfo(List<FileInfo> files, long term, long index) {
  this.termIndex = TermIndex.newTermIndex(term, index);
  this.files = Collections.unmodifiableList(files);
}
 

@Test
public void testApplyTransactionFailure() throws Exception {
  OzoneOutputStream key =
          objectStore.getVolume(volumeName).getBucket(bucketName)
                  .createKey("ratis", 1024, ReplicationType.RATIS,
                          ReplicationFactor.ONE, new HashMap<>());
  // First write and flush creates a container in the datanode
  key.write("ratis".getBytes());
  key.flush();
  key.write("ratis".getBytes());
  KeyOutputStream groupOutputStream = (KeyOutputStream) key.
          getOutputStream();
  List<OmKeyLocationInfo> locationInfoList =
          groupOutputStream.getLocationInfoList();
  Assert.assertEquals(1, locationInfoList.size());
  OmKeyLocationInfo omKeyLocationInfo = locationInfoList.get(0);
  HddsDatanodeService dn = TestHelper.getDatanodeService(omKeyLocationInfo,
          cluster);
  int index = cluster.getHddsDatanodeIndex(dn.getDatanodeDetails());
  ContainerData containerData = dn.getDatanodeStateMachine()
                  .getContainer().getContainerSet()
                  .getContainer(omKeyLocationInfo.getContainerID())
                  .getContainerData();
  Assert.assertTrue(containerData instanceof KeyValueContainerData);
  KeyValueContainerData keyValueContainerData =
          (KeyValueContainerData) containerData;
  key.close();
  ContainerStateMachine stateMachine =
      (ContainerStateMachine) TestHelper.getStateMachine(cluster.
          getHddsDatanodes().get(index), omKeyLocationInfo.getPipeline());
  SimpleStateMachineStorage storage =
          (SimpleStateMachineStorage) stateMachine.getStateMachineStorage();
  stateMachine.takeSnapshot();
  Path parentPath = storage.findLatestSnapshot().getFile().getPath();
  // Since the snapshot threshold is set to 1, since there are
  // applyTransactions, we should see snapshots
  Assert.assertTrue(parentPath.getParent().toFile().listFiles().length > 0);
  FileInfo snapshot = storage.findLatestSnapshot().getFile();
  Assert.assertNotNull(snapshot);
  long containerID = omKeyLocationInfo.getContainerID();
  // delete the container db file
  FileUtil.fullyDelete(new File(keyValueContainerData.getContainerPath()));
  Pipeline pipeline = cluster.getStorageContainerLocationClient()
          .getContainerWithPipeline(containerID).getPipeline();
  XceiverClientSpi xceiverClient =
          xceiverClientManager.acquireClient(pipeline);
  ContainerProtos.ContainerCommandRequestProto.Builder request =
          ContainerProtos.ContainerCommandRequestProto.newBuilder();
  request.setDatanodeUuid(pipeline.getFirstNode().getUuidString());
  request.setCmdType(ContainerProtos.Type.CloseContainer);
  request.setContainerID(containerID);
  request.setCloseContainer(
          ContainerProtos.CloseContainerRequestProto.getDefaultInstance());
  // close container transaction will fail over Ratis and will initiate
  // a pipeline close action

  try {
    xceiverClient.sendCommand(request.build());
    Assert.fail("Expected exception not thrown");
  } catch (IOException e) {
    // Exception should be thrown
  }
  // Make sure the container is marked unhealthy
  Assert.assertTrue(dn.getDatanodeStateMachine()
                  .getContainer().getContainerSet().getContainer(containerID)
                  .getContainerState()
                  == ContainerProtos.ContainerDataProto.State.UNHEALTHY);
  try {
    // try to take a new snapshot, ideally it should just fail
    stateMachine.takeSnapshot();
  } catch (IOException ioe) {
    Assert.assertTrue(ioe instanceof StateMachineException);
  }
  // Make sure the latest snapshot is same as the previous one
  FileInfo latestSnapshot = storage.findLatestSnapshot().getFile();
  Assert.assertTrue(snapshot.getPath().equals(latestSnapshot.getPath()));
}
 

public SingleFileSnapshotInfo(FileInfo fileInfo, long term, long endIndex) {
  super(Arrays.asList(fileInfo), term, endIndex);
}
 

@Override
public List<FileInfo> getFiles() {
  return files;
}
 

public FileListSnapshotInfo(List<FileInfo> files, long term, long index) {
  this.termIndex = TermIndex.newTermIndex(term, index);
  this.files = Collections.unmodifiableList(files);
}
 

@Override
public List<FileInfo> getFiles() {
  return null;
}
 

@Test
public void testApplyTransactionIdempotencyWithClosedContainer()
        throws Exception {
  OzoneOutputStream key =
          objectStore.getVolume(volumeName).getBucket(bucketName)
                  .createKey("ratis", 1024, ReplicationType.RATIS,
                          ReplicationFactor.ONE, new HashMap<>());
  // First write and flush creates a container in the datanode
  key.write("ratis".getBytes());
  key.flush();
  key.write("ratis".getBytes());
  KeyOutputStream groupOutputStream = (KeyOutputStream) key.getOutputStream();
  List<OmKeyLocationInfo> locationInfoList =
          groupOutputStream.getLocationInfoList();
  Assert.assertEquals(1, locationInfoList.size());
  OmKeyLocationInfo omKeyLocationInfo = locationInfoList.get(0);
  HddsDatanodeService dn = TestHelper.getDatanodeService(omKeyLocationInfo,
          cluster);
  ContainerData containerData = dn.getDatanodeStateMachine()
                  .getContainer().getContainerSet()
                  .getContainer(omKeyLocationInfo.getContainerID())
                  .getContainerData();
  Assert.assertTrue(containerData instanceof KeyValueContainerData);
  key.close();
  ContainerStateMachine stateMachine =
          (ContainerStateMachine) TestHelper.getStateMachine(dn,
                  omKeyLocationInfo.getPipeline());
  SimpleStateMachineStorage storage =
          (SimpleStateMachineStorage) stateMachine.getStateMachineStorage();
  Path parentPath = storage.findLatestSnapshot().getFile().getPath();
  stateMachine.takeSnapshot();
  Assert.assertTrue(parentPath.getParent().toFile().listFiles().length > 0);
  FileInfo snapshot = storage.findLatestSnapshot().getFile();
  Assert.assertNotNull(snapshot);
  long containerID = omKeyLocationInfo.getContainerID();
  Pipeline pipeline = cluster.getStorageContainerLocationClient()
          .getContainerWithPipeline(containerID).getPipeline();
  XceiverClientSpi xceiverClient =
          xceiverClientManager.acquireClient(pipeline);
  ContainerProtos.ContainerCommandRequestProto.Builder request =
          ContainerProtos.ContainerCommandRequestProto.newBuilder();
  request.setDatanodeUuid(pipeline.getFirstNode().getUuidString());
  request.setCmdType(ContainerProtos.Type.CloseContainer);
  request.setContainerID(containerID);
  request.setCloseContainer(
          ContainerProtos.CloseContainerRequestProto.getDefaultInstance());
  try {
    xceiverClient.sendCommand(request.build());
  } catch (IOException e) {
    Assert.fail("Exception should not be thrown");
  }
  Assert.assertTrue(
          TestHelper.getDatanodeService(omKeyLocationInfo, cluster)
                  .getDatanodeStateMachine()
                  .getContainer().getContainerSet().getContainer(containerID)
                  .getContainerState()
                  == ContainerProtos.ContainerDataProto.State.CLOSED);
  Assert.assertTrue(stateMachine.isStateMachineHealthy());
  try {
    stateMachine.takeSnapshot();
  } catch (IOException ioe) {
    Assert.fail("Exception should not be thrown");
  }
  FileInfo latestSnapshot = storage.findLatestSnapshot().getFile();
  Assert.assertFalse(snapshot.getPath().equals(latestSnapshot.getPath()));
}
 

/**
 * Returns a list of files corresponding to this snapshot. This list should include all
 * the files that the state machine keeps in its data directory. This list of files will be
 * copied as to other replicas in install snapshot RPCs.
 * @return a list of Files corresponding to the this snapshot.
 */
List<FileInfo> getFiles();
 

/**
 * Returns a list of files corresponding to this snapshot. This list should include all
 * the files that the state machine keeps in its data directory. This list of files will be
 * copied as to other replicas in install snapshot RPCs.
 * @return a list of Files corresponding to the this snapshot.
 */
List<FileInfo> getFiles();