org.apache.ratis.util.FileUtils Java Examples

/**
 * Verify transaction info with provided lastAppliedIndex.
 *
 * If transaction info transaction Index is less than or equal to
 * lastAppliedIndex, return false, else return true.
 * @param omTransactionInfo
 * @param lastAppliedIndex
 * @param leaderId
 * @param newDBlocation
 * @return boolean
 */
public static boolean verifyTransactionInfo(
    OMTransactionInfo omTransactionInfo,
    long lastAppliedIndex,
    String leaderId, Path newDBlocation) {
  if (omTransactionInfo.getTransactionIndex() <= lastAppliedIndex) {
    OzoneManager.LOG.error("Failed to install checkpoint from OM leader: {}" +
            ". The last applied index: {} is greater than or equal to the " +
            "checkpoint's applied index: {}. Deleting the downloaded " +
            "checkpoint {}", leaderId, lastAppliedIndex,
        omTransactionInfo.getTransactionIndex(), newDBlocation);
    try {
      FileUtils.deleteFully(newDBlocation);
    } catch (IOException e) {
      OzoneManager.LOG.error("Failed to fully delete the downloaded DB " +
          "checkpoint {} from OM leader {}.", newDBlocation, leaderId, e);
    }
    return false;
  }

  return true;
}
 

static void assertTruncatedLog(RaftPeerId id, File openLogFile, long lastIndex, MiniRaftCluster cluster) throws Exception {
  // truncate log
  FileUtils.truncateFile(openLogFile, openLogFile.length() - 1);
  final RaftServerImpl server = cluster.restartServer(id, false);
  // the last index should be one less than before
  Assert.assertEquals(lastIndex - 1, server.getState().getLog().getLastEntryTermIndex().getIndex());
  server.getProxy().close();
}
 

private RaftServerProxy newRaftServer(RaftPeerId id, RaftGroup group, boolean format) {
  LOG.info("newRaftServer: {}, {}, format? {}", id, group, format);
  try {
    final File dir = getStorageDir(id);
    if (format) {
      FileUtils.deleteFully(dir);
      LOG.info("Formatted directory {}", dir);
    }
    final RaftProperties prop = new RaftProperties(properties);
    RaftServerConfigKeys.setStorageDirs(prop, Collections.singletonList(dir));
    return newRaftServer(id, getStateMachineRegistry(properties), group, prop);
  } catch (IOException e) {
    throw new RuntimeException(e);
  }
}
 

/**
 * Check consistency of the storage directory.
 *
 * @return state {@link StorageState} of the storage directory
 */
StorageState analyzeStorage(boolean toLock) throws IOException {
  Objects.requireNonNull(root, "root directory is null");

  String rootPath = root.getCanonicalPath();
  try { // check that storage exists
    if (!root.exists()) {
      LOG.info("The storage directory " + rootPath + " does not exist. Creating ...");
      FileUtils.createDirectories(root);
    }
    // or is inaccessible
    if (!root.isDirectory()) {
      LOG.warn(rootPath + " is not a directory");
      return StorageState.NON_EXISTENT;
    }
    if (!Files.isWritable(root.toPath())) {
      LOG.warn("The storage directory " + rootPath + " is not writable.");
      return StorageState.NON_EXISTENT;
    }
  } catch(SecurityException ex) {
    LOG.warn("Cannot access storage directory " + rootPath, ex);
    return StorageState.NON_EXISTENT;
  }

  if (toLock) {
    this.lock(); // lock storage if it exists
  }

  // check whether current directory is valid
  if (hasMetaFile()) {
    return StorageState.NORMAL;
  } else {
    return StorageState.NOT_FORMATTED;
  }
}
 

private static void clearDirectory(File dir) throws IOException {
  if (dir.exists()) {
    LOG.info(dir + " already exists.  Deleting it ...");
    FileUtils.deleteFully(dir);
  }
  FileUtils.createDirectories(dir);
}
 

public LogOutputStream(File file, boolean append, long segmentMaxSize,
    long preallocatedSize, int bufferSize)
    throws IOException {
  this.file = file;
  this.checksum = new PureJavaCrc32C();
  this.segmentMaxSize = segmentMaxSize;
  this.preallocatedSize = preallocatedSize;
  RandomAccessFile rp = new RandomAccessFile(file, "rw");
  fc = rp.getChannel();
  fc.position(fc.size());
  preallocatedPos = fc.size();
  out = new BufferedWriteChannel(fc, bufferSize);

  try {
    fc = rp.getChannel();
    fc.position(fc.size());
    preallocatedPos = fc.size();

    out = new BufferedWriteChannel(fc, bufferSize);
    if (!append) {
      create();
    }
  } catch (IOException ioe) {
    LOG.warn("Hit IOException while creating log segment " + file
        + ", delete the partial file.");
    // hit IOException, clean up the in-progress log file
    try {
      FileUtils.deleteFully(file);
    } catch (IOException e) {
      LOG.warn("Failed to delete the file " + file, e);
    }
    throw ioe;
  }
}
 

static LogSegment loadSegment(RaftStorage storage, File file,
    long start, long end, boolean isOpen,
    boolean keepEntryInCache, Consumer<LogEntryProto> logConsumer)
    throws IOException {
  final LogSegment segment = isOpen ?
      LogSegment.newOpenSegment(storage, start) :
      LogSegment.newCloseSegment(storage, start, end);

  final int entryCount = readSegmentFile(file, start, end, isOpen, entry -> {
    segment.append(keepEntryInCache || isOpen, entry);
    if (logConsumer != null) {
      logConsumer.accept(entry);
    }
  });
  LOG.info("Successfully read {} entries from segment file {}", entryCount, file);

  if (entryCount == 0) {
    // The segment does not have any entries, delete the file.
    FileUtils.deleteFile(file);
    return null;
  } else if (file.length() > segment.getTotalSize()) {
    // The segment has extra padding, truncate it.
    FileUtils.truncateFile(file, segment.getTotalSize());
  }

  Preconditions.assertTrue(start == segment.getStartIndex());
  if (!segment.records.isEmpty()) {
    Preconditions.assertTrue(start == segment.records.get(0).getTermIndex().getIndex());
  }
  if (!isOpen) {
    Preconditions.assertTrue(segment.getEndIndex() == end);
  }
  return segment;
}
 

CompletableFuture<Path> delete(long index, String relative) {
  final Supplier<String> name = () -> "delete(" + relative + ") @" + getId() + ":" + index;
  final CheckedSupplier<Path, IOException> task = LogUtils.newCheckedSupplier(LOG, () -> {
    final FileInfo info = files.remove(relative);
    FileUtils.delete(resolve(info.getRelativePath()));
    return info.getRelativePath();
  }, name);
  return submit(task, deleter);
}
 

@Override
public void initialize(RaftServer server, RaftGroupId groupId, RaftStorage raftStorage)
    throws IOException {
  super.initialize(server, groupId, raftStorage);
  this.storage.init(raftStorage);
  FileUtils.createDirectories(files.getRoot());
}
 

File prepareLog(boolean isOpen, long startIndex, int numEntries, long term, boolean isLastEntryPartiallyWritten)
    throws IOException {
  if (!isOpen) {
    Preconditions.assertTrue(!isLastEntryPartiallyWritten, "For closed log, the last entry cannot be partially written.");
  }
  RaftStorage storage = new RaftStorage(storageDir, StartupOption.REGULAR);
  final File file = isOpen ?
      storage.getStorageDir().getOpenLogFile(startIndex) :
      storage.getStorageDir().getClosedLogFile(startIndex, startIndex + numEntries - 1);

  final LogEntryProto[] entries = new LogEntryProto[numEntries];
  try (LogOutputStream out = new LogOutputStream(file, false,
      segmentMaxSize, preallocatedSize, bufferSize)) {
    for (int i = 0; i < entries.length; i++) {
      SimpleOperation op = new SimpleOperation("m" + i);
      entries[i] = ServerProtoUtils.toLogEntryProto(op.getLogEntryContent(), term, i + startIndex);
      out.write(entries[i]);
    }
  }

  if (isLastEntryPartiallyWritten) {
    final int entrySize = size(entries[entries.length - 1]);
    final int truncatedEntrySize = ThreadLocalRandom.current().nextInt(entrySize - 1) + 1;
    // 0 < truncatedEntrySize < entrySize
    final long fileLength = file.length();
    final long truncatedFileLength = fileLength - (entrySize - truncatedEntrySize);
    LOG.info("truncate last entry: entry(size={}, truncated={}), file(length={}, truncated={})",
        entrySize, truncatedEntrySize, fileLength, truncatedFileLength);
    FileUtils.truncateFile(file, truncatedFileLength);
  }

  storage.close();
  return file;
}
 

/**
 * check if RaftStorage deletes tmp metafile when startup
 */
@Test
public void testCleanMetaTmpFile() throws Exception {
  RaftStorage storage = new RaftStorage(storageDir, StartupOption.REGULAR);
  Assert.assertEquals(StorageState.NORMAL, storage.getState());
  storage.close();

  RaftStorageDirectory sd = new RaftStorageDirectory(storageDir);
  File metaFile = sd.getMetaFile();
  FileUtils.move(metaFile, sd.getMetaTmpFile());

  Assert.assertEquals(StorageState.NOT_FORMATTED, sd.analyzeStorage(false));

  try {
    new RaftStorage(storageDir, StartupOption.REGULAR);
    Assert.fail("should throw IOException since storage dir is not formatted");
  } catch (IOException e) {
    Assert.assertTrue(
        e.getMessage().contains(StorageState.NOT_FORMATTED.name()));
  }

  // let the storage dir contain both raft-meta and raft-meta.tmp
  new RaftStorage(storageDir, StartupOption.FORMAT).close();
  Assert.assertTrue(sd.getMetaFile().exists());
  Assert.assertTrue(sd.getMetaTmpFile().createNewFile());
  Assert.assertTrue(sd.getMetaTmpFile().exists());
  try {
    storage = new RaftStorage(storageDir, StartupOption.REGULAR);
    Assert.assertEquals(StorageState.NORMAL, storage.getState());
    Assert.assertFalse(sd.getMetaTmpFile().exists());
    Assert.assertTrue(sd.getMetaFile().exists());
  } finally {
    storage.close();
  }
}
 

private RaftServerProxy newRaftServer(RaftPeerId id, RaftGroup group, boolean format) {
  LOG.info("newRaftServer: {}, {}, format? {}", id, group, format);
  try {
    final File dir = getStorageDir(id);
    if (format) {
      FileUtils.deleteFully(dir);
      LOG.info("Formatted directory {}", dir);
    }
    final RaftProperties prop = new RaftProperties(properties);
    RaftServerConfigKeys.setStorageDir(prop, Collections.singletonList(dir));
    return newRaftServer(id, getStateMachineRegistry(properties), group, prop);
  } catch (IOException e) {
    throw new RuntimeException(e);
  }
}
 

/**
 * Lock storage to provide exclusive access.
 *
 * <p> Locking is not supported by all file systems.
 * E.g., NFS does not consistently support exclusive locks.
 *
 * <p> If locking is supported we guarantee exclusive access to the
 * storage directory. Otherwise, no guarantee is given.
 *
 * @throws IOException if locking fails
 */
public void lock() throws IOException {
  final File lockF = new File(root, STORAGE_FILE_LOCK);
  final FileLock newLock = FileUtils.attempt(() -> tryLock(lockF), () -> "tryLock " + lockF);
  if (newLock == null) {
    String msg = "Cannot lock storage " + this.root
        + ". The directory is already locked";
    LOG.info(msg);
    throw new IOException(msg);
  }
  // Don't overwrite lock until success - this way if we accidentally
  // call lock twice, the internal state won't be cleared by the second
  // (failed) lock attempt
  lock = newLock;
}
 

/**
 * Check consistency of the storage directory.
 *
 * @return state {@link StorageState} of the storage directory
 */
StorageState analyzeStorage(boolean toLock) throws IOException {
  Objects.requireNonNull(root, "root directory is null");

  String rootPath = root.getCanonicalPath();
  try { // check that storage exists
    if (!root.exists()) {
      LOG.info("The storage directory " + rootPath + " does not exist. Creating ...");
      FileUtils.createDirectories(root);
    }
    // or is inaccessible
    if (!root.isDirectory()) {
      LOG.warn(rootPath + " is not a directory");
      return StorageState.NON_EXISTENT;
    }
    if (!Files.isWritable(root.toPath())) {
      LOG.warn("The storage directory " + rootPath + " is not writable.");
      return StorageState.NON_EXISTENT;
    }
  } catch(SecurityException ex) {
    LOG.warn("Cannot access storage directory " + rootPath, ex);
    return StorageState.NON_EXISTENT;
  }

  if (toLock) {
    this.lock(); // lock storage if it exists
  }

  // check whether current directory is valid
  if (hasMetaFile()) {
    return StorageState.NORMAL;
  } else {
    return StorageState.NOT_FORMATTED;
  }
}
 

private static void clearDirectory(File dir) throws IOException {
  if (dir.exists()) {
    LOG.info(dir + " already exists.  Deleting it ...");
    FileUtils.deleteFully(dir);
  }
  FileUtils.createDirectories(dir);
}
 

CompletableFuture<Path> delete(long index, String relative) {
  final Supplier<String> name = () -> "delete(" + relative + ") @" + getId() + ":" + index;
  final CheckedSupplier<Path, IOException> task = LogUtils.newCheckedSupplier(LOG, () -> {
    final FileInfo info = files.remove(relative);
    FileUtils.delete(resolve(info.getRelativePath()));
    return info.getRelativePath();
  }, name);
  return submit(task, deleter);
}
 

@Override
public void initialize(RaftServer server, RaftGroupId groupId, RaftStorage raftStorage)
    throws IOException {
  super.initialize(server, groupId, raftStorage);
  this.storage.init(raftStorage);
  FileUtils.createDirectories(files.getRoot());
}
 

File prepareLog(boolean isOpen, long startIndex, int numEntries, long term, boolean isLastEntryPartiallyWritten)
    throws IOException {
  if (!isOpen) {
    Preconditions.assertTrue(!isLastEntryPartiallyWritten, "For closed log, the last entry cannot be partially written.");
  }
  RaftStorage storage = new RaftStorage(storageDir, StartupOption.REGULAR);
  final File file = isOpen ?
      storage.getStorageDir().getOpenLogFile(startIndex) :
      storage.getStorageDir().getClosedLogFile(startIndex, startIndex + numEntries - 1);

  final LogEntryProto[] entries = new LogEntryProto[numEntries];
  try (SegmentedRaftLogOutputStream out = new SegmentedRaftLogOutputStream(file, false,
      segmentMaxSize, preallocatedSize, ByteBuffer.allocateDirect(bufferSize))) {
    for (int i = 0; i < entries.length; i++) {
      SimpleOperation op = new SimpleOperation("m" + i);
      entries[i] = ServerProtoUtils.toLogEntryProto(op.getLogEntryContent(), term, i + startIndex);
      out.write(entries[i]);
    }
  }

  if (isLastEntryPartiallyWritten) {
    final int entrySize = size(entries[entries.length - 1]);
    final int truncatedEntrySize = ThreadLocalRandom.current().nextInt(entrySize - 1) + 1;
    // 0 < truncatedEntrySize < entrySize
    final long fileLength = file.length();
    final long truncatedFileLength = fileLength - (entrySize - truncatedEntrySize);
    LOG.info("truncate last entry: entry(size={}, truncated={}), file(length={}, truncated={})",
        entrySize, truncatedEntrySize, fileLength, truncatedFileLength);
    FileUtils.truncateFile(file, truncatedFileLength);
  }

  storage.close();
  return file;
}
 

static void assertTruncatedLog(RaftPeerId id, File openLogFile, long lastIndex, MiniRaftCluster cluster) throws Exception {
  // truncate log
  FileUtils.truncateFile(openLogFile, openLogFile.length() - 1);
  final RaftServerImpl server = cluster.restartServer(id, false);
  // the last index should be one less than before
  Assert.assertEquals(lastIndex - 1, server.getState().getLog().getLastEntryTermIndex().getIndex());
  server.getProxy().close();
}
 

@Override
public void cleanupOldSnapshots(SnapshotRetentionPolicy snapshotRetentionPolicy) throws IOException {
  if (snapshotRetentionPolicy != null && snapshotRetentionPolicy.getNumSnapshotsRetained() > 0) {

    List<SingleFileSnapshotInfo> allSnapshotFiles = new ArrayList<>();
    try (DirectoryStream<Path> stream =
             Files.newDirectoryStream(smDir.toPath())) {
      for (Path path : stream) {
        Matcher matcher = SNAPSHOT_REGEX.matcher(path.getFileName().toString());
        if (matcher.matches()) {
          final long endIndex = Long.parseLong(matcher.group(2));
          final long term = Long.parseLong(matcher.group(1));
          final FileInfo fileInfo = new FileInfo(path, null); //We don't need FileDigest here.
          allSnapshotFiles.add(new SingleFileSnapshotInfo(fileInfo, term, endIndex));
        }
      }
    }

    if (allSnapshotFiles.size() > snapshotRetentionPolicy.getNumSnapshotsRetained()) {
      allSnapshotFiles.sort(new SnapshotFileComparator());
      List<File> snapshotFilesToBeCleaned = allSnapshotFiles.subList(
          snapshotRetentionPolicy.getNumSnapshotsRetained(), allSnapshotFiles.size()).stream()
          .map(singleFileSnapshotInfo -> singleFileSnapshotInfo.getFile().getPath().toFile())
          .collect(Collectors.toList());
      for (File snapshotFile : snapshotFilesToBeCleaned) {
        LOG.info("Deleting old snapshot at {}", snapshotFile.getAbsolutePath());
        FileUtils.deleteFileQuietly(snapshotFile);
      }
    }
  }
}
 

/**
 * check if RaftStorage deletes tmp metafile when startup
 */
@Test
public void testCleanMetaTmpFile() throws Exception {
  RaftStorage storage = new RaftStorage(storageDir, StartupOption.REGULAR);
  Assert.assertEquals(StorageState.NORMAL, storage.getState());
  storage.close();

  RaftStorageDirectory sd = new RaftStorageDirectory(storageDir);
  File metaFile = sd.getMetaFile();
  FileUtils.move(metaFile, sd.getMetaTmpFile());

  Assert.assertEquals(StorageState.NOT_FORMATTED, sd.analyzeStorage(false));

  try {
    new RaftStorage(storageDir, StartupOption.REGULAR);
    Assert.fail("should throw IOException since storage dir is not formatted");
  } catch (IOException e) {
    Assert.assertTrue(
        e.getMessage().contains(StorageState.NOT_FORMATTED.name()));
  }

  // let the storage dir contain both raft-meta and raft-meta.tmp
  new RaftStorage(storageDir, StartupOption.FORMAT).close();
  Assert.assertTrue(sd.getMetaFile().exists());
  Assert.assertTrue(sd.getMetaTmpFile().createNewFile());
  Assert.assertTrue(sd.getMetaTmpFile().exists());
  try {
    storage = new RaftStorage(storageDir, StartupOption.REGULAR);
    Assert.assertEquals(StorageState.NORMAL, storage.getState());
    Assert.assertFalse(sd.getMetaTmpFile().exists());
    Assert.assertTrue(sd.getMetaFile().exists());
  } finally {
    storage.close();
  }
}
 

@After
public void tearDown() throws Exception {
  if (storageDir != null) {
    FileUtils.deleteFully(storageDir.getParentFile());
  }
}
 

@AfterClass
public static void shutdown() throws IOException {
  cluster.shutdown();
  FileUtils.deleteFully(new File(dbPath));
}
 

/**
 * Basic test for install snapshot: start a one node cluster and let it
 * generate a snapshot. Then delete the log and restart the node, and add more
 * nodes as followers.
 */
@Test
public void testBasicInstallSnapshot() throws Exception {
  final List<LogPathAndIndex> logs;
  try {
    RaftTestUtil.waitForLeader(cluster);
    final RaftPeerId leaderId = cluster.getLeader().getId();

    int i = 0;
    try(final RaftClient client = cluster.createClient(leaderId)) {
      for (; i < SNAPSHOT_TRIGGER_THRESHOLD * 2 - 1; i++) {
        RaftClientReply reply = client.send(new SimpleMessage("m" + i));
        Assert.assertTrue(reply.isSuccess());
      }
    }

    // wait for the snapshot to be done
    RaftStorageDirectory storageDirectory = cluster.getLeader().getState()
        .getStorage().getStorageDir();

    final long nextIndex = cluster.getLeader().getState().getLog().getNextIndex();
    LOG.info("nextIndex = {}", nextIndex);
    final List<File> snapshotFiles = getSnapshotFiles(cluster, nextIndex - SNAPSHOT_TRIGGER_THRESHOLD, nextIndex);
    JavaUtils.attempt(() -> snapshotFiles.stream().anyMatch(RaftSnapshotBaseTest::exists),
        10, 1000, "snapshotFile.exist", LOG);
    logs = storageDirectory.getLogSegmentFiles();
  } finally {
    cluster.shutdown();
  }

  // delete the log segments from the leader
  for (LogPathAndIndex path : logs) {
    FileUtils.delete(path.getPath());
  }

  // restart the peer
  LOG.info("Restarting the cluster");
  cluster.restart(false);
  try {
    assertLeaderContent(cluster);

    // generate some more traffic
    try(final RaftClient client = cluster.createClient(cluster.getLeader().getId())) {
      Assert.assertTrue(client.send(new SimpleMessage("test")).isSuccess());
    }

    // add two more peers
    MiniRaftCluster.PeerChanges change = cluster.addNewPeers(
        new String[]{"s3", "s4"}, true);
    // trigger setConfiguration
    cluster.setConfiguration(change.allPeersInNewConf);

    RaftServerTestUtil.waitAndCheckNewConf(cluster, change.allPeersInNewConf, 0, null);
  } finally {
    cluster.shutdown();
  }
}
 

@After
public void tearDown() throws Exception {
  if (storageDir != null) {
    FileUtils.deleteFully(storageDir.getParentFile());
  }
}
 

@After
public void tearDown() throws Exception {
  if (storageDir != null) {
    FileUtils.deleteFully(storageDir.getParentFile());
  }
}
 

@After
public void tearDown() throws Exception {
  if (storageDir != null) {
    FileUtils.deleteFully(storageDir.getParentFile());
  }
}
 

@AfterClass
public static void tearDown() throws IOException {
  FileUtils.deleteFully(rootTestDir.get());
}
 

public void cleanUp() throws IOException {
    FileUtils.deleteFully(new File(getServerOpts().getWorkingDir()));
}
 

FileOut(Path p) throws IOException {
  this.out = FileUtils.createNewFile(p);
  this.channel = Channels.newChannel(out);
}