org.apache.bookkeeper.util.IOUtils Java Examples

private void createSegmentStoreFileSystem() throws IOException {
    File rootDir = this.segmentStoreRoot.get();
    if (rootDir == null || !rootDir.exists()) {
        rootDir = IOUtils.createTempDir("selftest.segmentstore.", "");
        rootDir.deleteOnExit();
        this.segmentStoreRoot.set(rootDir);
    }

    // Config file.
    File configFile = new File(getSegmentStoreConfigFilePath());
    configFile.delete();
    configFile.createNewFile();
    log("SegmentStore Config: '%s'.", configFile.getAbsolutePath());

    // Storage.
    File storageDir = new File(getSegmentStoreStoragePath());
    storageDir.delete();
    storageDir.mkdir();
    log("SegmentStore Storage: '%s/'.", storageDir.getAbsolutePath());
    this.builderConfig.store(configFile);
}
 

@Test(timeout = 60000)
public void testRunZookeeperOnAnyPort() throws Exception {
    Pair<ZooKeeperServerShim, Integer> serverAndPort1 = null;
    Pair<ZooKeeperServerShim, Integer> serverAndPort2 = null;
    Pair<ZooKeeperServerShim, Integer> serverAndPort3 = null;
    try {
        File zkTmpDir1 = IOUtils.createTempDir("zookeeper1", "distrlog");
        serverAndPort1 = LocalDLMEmulator.runZookeeperOnAnyPort(7000, zkTmpDir1);
        File zkTmpDir2 = IOUtils.createTempDir("zookeeper2", "distrlog");
        serverAndPort2 = LocalDLMEmulator.runZookeeperOnAnyPort(7000, zkTmpDir2);
        File zkTmpDir3 = IOUtils.createTempDir("zookeeper3", "distrlog");
        serverAndPort3 = LocalDLMEmulator.runZookeeperOnAnyPort(7000, zkTmpDir3);
    } catch (Exception ex) {
        if (null != serverAndPort1) {
            serverAndPort1.getLeft().stop();
        }
        if (null != serverAndPort2) {
            serverAndPort2.getLeft().stop();
        }
        if (null != serverAndPort3) {
            serverAndPort3.getLeft().stop();
        }
    }
}
 

@Override
protected int runCmd() throws Exception {
    MetadataUpdater metadataUpdater = dryrun
            ? new DryrunLogSegmentMetadataStoreUpdater(getConf(),
                    getLogSegmentMetadataStore()) :
            LogSegmentMetadataStoreUpdater.createMetadataUpdater(getConf(),
                    getLogSegmentMetadataStore());
    System.out.println("List of streams : ");
    System.out.println(streams);
    if (!IOUtils.confirmPrompt("Do you want to repair all these streams (Y/N):")) {
        return -1;
    }
    for (String stream : streams) {
        fixInprogressSegmentWithLowerSequenceNumber(getNamespace(),
                metadataUpdater, stream, verbose, !getForce());
    }
    return 0;
}
 

@BeforeClass
public static void setupCluster() throws Exception {
    File zkTmpDir = IOUtils.createTempDir("zookeeper", "distrlog");
    tmpDirs.add(zkTmpDir);
    Pair<ZooKeeperServerShim, Integer> serverAndPort = LocalDLMEmulator.runZookeeperOnAnyPort(zkTmpDir);
    zks = serverAndPort.getLeft();
    zkPort = serverAndPort.getRight();
    bkutil = LocalDLMEmulator.newBuilder()
            .numBookies(numBookies)
            .zkHost("127.0.0.1")
            .zkPort(zkPort)
            .serverConf(DLMTestUtil.loadTestBkConf())
            .shouldStartZK(false)
            .build();
    bkutil.start();
    zkServers = "127.0.0.1:" + zkPort;
}
 

private int truncateStream(final com.twitter.distributedlog.DistributedLogManagerFactory factory, String streamName, DLSN dlsn) throws Exception {
    DistributedLogManager dlm = factory.createDistributedLogManagerWithSharedClients(streamName);
    try {
        long totalRecords = dlm.getLogRecordCount();
        long recordsAfterTruncate = Await.result(dlm.getLogRecordCountAsync(dlsn));
        long recordsToTruncate = totalRecords - recordsAfterTruncate;
        if (!getForce() && !IOUtils.confirmPrompt("Do you want to truncate " + streamName + " at dlsn " + dlsn + " (" + recordsToTruncate + " records)?")) {
            return 0;
        } else {
            AsyncLogWriter writer = dlm.startAsyncLogSegmentNonPartitioned();
            try {
                if (!Await.result(writer.truncate(dlsn))) {
                    System.out.println("Failed to truncate.");
                }
                return 0;
            } finally {
                Utils.close(writer);
            }
        }
    } catch (Exception ex) {
        System.err.println("Failed to truncate " + ex);
        return 1;
    } finally {
        dlm.close();
    }
}
 

@Override
protected int runCmd() throws Exception {
    if (!getForce() && !IOUtils.confirmPrompt("Do you want to audit uris : "
            + getUris() + ", allocation paths = " + allocationPaths)) {
        return 0;
    }

    DLAuditor dlAuditor = new DLAuditor(getConf());
    try {
        Pair<Set<Long>, Set<Long>> bkdlLedgers = dlAuditor.collectLedgers(getUris(), allocationPaths);
        dumpLedgers(bkdlLedgers.getLeft(), new File(ledgersFilePrefix + "-bkledgers.txt"));
        dumpLedgers(bkdlLedgers.getRight(), new File(ledgersFilePrefix + "-dlledgers.txt"));
        dumpLedgers(Sets.difference(bkdlLedgers.getLeft(), bkdlLedgers.getRight()),
                    new File(ledgersFilePrefix + "-leakledgers.txt"));
    } finally {
        dlAuditor.close();
    }
    return 0;
}
 

@Override
protected int runCmd() throws Exception {
    generateStreams(streamPrefix, streamExpression);
    if (streams.isEmpty()) {
        System.out.println("Nothing to create.");
        return 0;
    }
    if (!getForce() && !IOUtils.confirmPrompt("You are going to create streams : " + streams)) {
        return 0;
    }
    getConf().setZkAclId(getZkAclId());
    for (String stream : streams) {
        getNamespace().createLog(stream);
    }
    return 0;
}
 

@Test(timeout = 60000)
public void testRunZookeeperOnAnyPort() throws Exception {
    Pair<ZooKeeperServerShim, Integer> serverAndPort1 = null;
    Pair<ZooKeeperServerShim, Integer> serverAndPort2 = null;
    Pair<ZooKeeperServerShim, Integer> serverAndPort3 = null;
    try {
        File zkTmpDir1 = IOUtils.createTempDir("zookeeper1", "distrlog");
        serverAndPort1 = LocalDLMEmulator.runZookeeperOnAnyPort(7000, zkTmpDir1);
        File zkTmpDir2 = IOUtils.createTempDir("zookeeper2", "distrlog");
        serverAndPort2 = LocalDLMEmulator.runZookeeperOnAnyPort(7000, zkTmpDir2);
        File zkTmpDir3 = IOUtils.createTempDir("zookeeper3", "distrlog");
        serverAndPort3 = LocalDLMEmulator.runZookeeperOnAnyPort(7000, zkTmpDir3);
    } catch (Exception ex) {
        if (null != serverAndPort1) {
            serverAndPort1.getLeft().stop();
        }
        if (null != serverAndPort2) {
            serverAndPort2.getLeft().stop();
        }
        if (null != serverAndPort3) {
            serverAndPort3.getLeft().stop();
        }
    }
}
 

@BeforeClass
public static void setupCluster() throws Exception {
    File zkTmpDir = IOUtils.createTempDir("zookeeper", "distrlog");
    tmpDirs.add(zkTmpDir);
    Pair<ZooKeeperServerShim, Integer> serverAndPort = LocalDLMEmulator.runZookeeperOnAnyPort(zkTmpDir);
    zks = serverAndPort.getLeft();
    zkPort = serverAndPort.getRight();
    bkutil = LocalDLMEmulator.newBuilder()
            .numBookies(numBookies)
            .zkHost("127.0.0.1")
            .zkPort(zkPort)
            .serverConf(DLMTestUtil.loadTestBkConf())
            .shouldStartZK(false)
            .build();
    bkutil.start();
    zkServers = "127.0.0.1:" + zkPort;
    Thread.setDefaultUncaughtExceptionHandler(new Thread.UncaughtExceptionHandler() {
        @Override
        public void uncaughtException(Thread t, Throwable e) {
            LOG.warn("Uncaught exception at Thread {} : ", t.getName(), e);
        }
    });
}
 

@Override
protected int runCmd() throws Exception {
    if (!getForce() && !IOUtils.confirmPrompt("Do you want to audit uris : "
            + getUris() + ", allocation paths = " + allocationPaths)) {
        return 0;
    }

    DLAuditor dlAuditor = new DLAuditor(getConf());
    try {
        Pair<Set<Long>, Set<Long>> bkdlLedgers = dlAuditor.collectLedgers(getUris(), allocationPaths);
        dumpLedgers(bkdlLedgers.getLeft(), new File(ledgersFilePrefix + "-bkledgers.txt"));
        dumpLedgers(bkdlLedgers.getRight(), new File(ledgersFilePrefix + "-dlledgers.txt"));
        dumpLedgers(Sets.difference(bkdlLedgers.getLeft(), bkdlLedgers.getRight()),
                    new File(ledgersFilePrefix + "-leakledgers.txt"));
    } finally {
        dlAuditor.close();
    }
    return 0;
}
 

@Override
protected int runCmd() throws Exception {
    generateStreams(streamPrefix, streamExpression);
    if (streams.isEmpty()) {
        System.out.println("Nothing to create.");
        return 0;
    }
    if (!getForce() && !IOUtils.confirmPrompt("You are going to create streams : " + streams)) {
        return 0;
    }
    getConf().setZkAclId(getZkAclId());
    for (String stream : streams) {
        getFactory().getNamespace().createLog(stream);
    }
    return 0;
}
 

@Override
protected int runCmd() throws Exception {
    MetadataUpdater metadataUpdater = dryrun ?
            new DryrunLogSegmentMetadataStoreUpdater(getConf(),
                    getLogSegmentMetadataStore()) :
            LogSegmentMetadataStoreUpdater.createMetadataUpdater(getConf(),
                    getLogSegmentMetadataStore());
    System.out.println("List of streams : ");
    System.out.println(streams);
    if (!IOUtils.confirmPrompt("Do you want to repair all these streams (Y/N):")) {
        return -1;
    }
    for (String stream : streams) {
        fixInprogressSegmentWithLowerSequenceNumber(getFactory(), metadataUpdater, stream, verbose, !getForce());
    }
    return 0;
}
 

protected void repairLogSegment(BookKeeperAdmin bkAdmin,
                                MetadataUpdater metadataUpdater,
                                LogSegmentMetadata segment) throws Exception {
    if (segment.isInProgress()) {
        System.out.println("Skip inprogress log segment " + segment);
        return;
    }
    LedgerHandle lh = bkAdmin.openLedger(segment.getLedgerId(), true);
    long lac = lh.getLastAddConfirmed();
    Enumeration<LedgerEntry> entries = lh.readEntries(lac, lac);
    if (!entries.hasMoreElements()) {
        throw new IOException("Entry " + lac + " isn't found for " + segment);
    }
    LedgerEntry lastEntry = entries.nextElement();
    Entry.Reader reader = Entry.newBuilder()
            .setLogSegmentInfo(segment.getLogSegmentSequenceNumber(), segment.getStartSequenceId())
            .setEntryId(lastEntry.getEntryId())
            .setEnvelopeEntry(LogSegmentMetadata.supportsEnvelopedEntries(segment.getVersion()))
            .setInputStream(lastEntry.getEntryInputStream())
            .buildReader();
    LogRecordWithDLSN record = reader.nextRecord();
    LogRecordWithDLSN lastRecord = null;
    while (null != record) {
        lastRecord = record;
        record = reader.nextRecord();
    }
    if (null == lastRecord) {
        throw new IOException("No record found in entry " + lac + " for " + segment);
    }
    System.out.println("Updating last record for " + segment + " to " + lastRecord);
    if (!IOUtils.confirmPrompt("Do you want to make this change (Y/N): ")) {
        return;
    }
    metadataUpdater.updateLastRecord(segment, lastRecord);
}
 

@Override
protected int runCmd(CommandLine cmdline) throws Exception {
    String[] args = cmdline.getArgs();
    if (args.length <= 0) {
        System.err.println("No distributedlog uri specified.");
        printUsage();
        return -1;
    }
    boolean force = cmdline.hasOption("f");
    URI uri = URI.create(args[0]);
    // resolving the uri to see if there is another bindings in this uri.
    ZooKeeperClient zkc = ZooKeeperClientBuilder.newBuilder().uri(uri)
            .sessionTimeoutMs(10000).build();
    BKDLConfig bkdlConfig;
    try {
        bkdlConfig = BKDLConfig.resolveDLConfig(zkc, uri);
    } catch (IOException ie) {
        bkdlConfig = null;
    }
    if (null == bkdlConfig) {
        System.out.println("No bookkeeper is bound to " + uri);
        return 0;
    } else {
        System.out.println("There is bookkeeper bound to " + uri + " : ");
        System.out.println("");
        System.out.println(bkdlConfig.toString());
        System.out.println("");
        if (!force && !IOUtils.confirmPrompt("Do you want to unbind " + uri + " :\n")) {
            return 0;
        }
    }
    DLMetadata.unbind(uri);
    System.out.println("Unbound on " + uri + ".");
    return 0;
}
 

public static void main(String[] args) throws Exception {
    try {
        if (args.length < 1) {
            System.out.println("Usage: LocalDLEmulator <zk_port>");
            System.exit(-1);
        }

        final int zkPort = Integer.parseInt(args[0]);
        final File zkDir = IOUtils.createTempDir("distrlog", "zookeeper");
        final LocalDLMEmulator localDlm = LocalDLMEmulator.newBuilder()
            .zkPort(zkPort)
            .build();

        Runtime.getRuntime().addShutdownHook(new Thread() {
            @Override
            public void run() {
                try {
                    localDlm.teardown();
                    FileUtils.deleteDirectory(zkDir);
                    System.out.println("ByeBye!");
                } catch (Exception e) {
                    // do nothing
                }
            }
        });
        localDlm.start();

        System.out.println(String.format(
            "DistributedLog Sandbox is running now. You could access distributedlog://%s:%s",
            DEFAULT_ZK_HOST,
            zkPort));
    } catch (Exception ex) {
        System.out.println("Exception occurred running emulator " + ex);
    }
}
 

protected int inspectAndRepair(List<LogSegmentMetadata> segments) throws Exception {
    LogSegmentMetadataStore metadataStore = getLogSegmentMetadataStore();
    ZooKeeperClient zkc = getZooKeeperClient();
    BKDLConfig bkdlConfig = BKDLConfig.resolveDLConfig(zkc, getUri());
    BKDLConfig.propagateConfiguration(bkdlConfig, getConf());
    BookKeeperClient bkc = BookKeeperClientBuilder.newBuilder()
            .dlConfig(getConf())
            .zkServers(bkdlConfig.getBkZkServersForReader())
            .ledgersPath(bkdlConfig.getBkLedgersPath())
            .name("dlog")
            .build();
    try {
        List<LogSegmentMetadata> segmentsToRepair = inspectLogSegments(bkc, segments);
        if (segmentsToRepair.isEmpty()) {
            System.out.println("The stream is good. No log segments to repair.");
            return 0;
        }
        System.out.println(segmentsToRepair.size() + " segments to repair : ");
        System.out.println(segmentsToRepair);
        System.out.println();
        if (!IOUtils.confirmPrompt("Do you want to repair them (Y/N): ")) {
            return 0;
        }
        repairLogSegments(metadataStore, bkc, segmentsToRepair);
        return 0;
    } finally {
        bkc.close();
    }
}
 

private BookieServer runBookie(int bkPort) throws Exception {
    // Attempt to reuse an existing data directory. This is useful in case of stops & restarts, when we want to preserve
    // already committed data.
    File journalDir = this.journalDirs.getOrDefault(bkPort, null);
    if (journalDir == null) {
        journalDir = IOUtils.createTempDir("bookiejournal_" + bkPort, "_test");
        log.info("Journal Dir[{}]: {}.", bkPort, journalDir.getPath());
        this.journalDirs.put(bkPort, journalDir);
        setupTempDir(journalDir);
    }

    File ledgerDir = this.ledgerDirs.getOrDefault(bkPort, null);
    if (ledgerDir == null) {
        ledgerDir = Strings.isNullOrEmpty(this.ledgersDir) ? null : new File(this.ledgersDir);
        ledgerDir = IOUtils.createTempDir("bookieledger_" + bkPort, "_test", ledgerDir);
        log.info("Ledgers Dir[{}]: {}.", bkPort, ledgerDir.getPath());
        this.ledgerDirs.put(bkPort, ledgerDir);
        setupTempDir(ledgerDir);
    }

    val conf = new ServerConfiguration();
    conf.setBookiePort(bkPort);
    conf.setMetadataServiceUri("zk://" + LOOPBACK_ADDRESS.getHostAddress() + ":" + this.zkPort + ledgersPath);
    conf.setJournalDirName(journalDir.getPath());
    conf.setLedgerDirNames(new String[]{ledgerDir.getPath()});
    conf.setAllowLoopback(true);
    conf.setJournalAdaptiveGroupWrites(true);

    if (secureBK) {
        conf.setTLSProvider("OpenSSL");
        conf.setTLSProviderFactoryClass("org.apache.bookkeeper.tls.TLSContextFactory");
        conf.setTLSKeyStore(this.tLSKeyStore);
        conf.setTLSKeyStorePasswordPath(this.tLSKeyStorePasswordPath);
    }

    log.info("Starting Bookie at port " + bkPort);
    val bs = new BookieServer(conf);
    bs.start();
    return bs;
}
 

@BeforeClass
public static void setupZooKeeper() throws Exception {
    zkDir = IOUtils.createTempDir("zookeeper", ZooKeeperClusterTestCase.class.getName());
    Pair<ZooKeeperServerShim, Integer> serverAndPort = LocalDLMEmulator.runZookeeperOnAnyPort(zkDir);
    zks = serverAndPort.getLeft();
    zkPort = serverAndPort.getRight();
    zkServers = "127.0.0.1:" + zkPort;
}
 

private DistributedLogCluster(DistributedLogConfiguration dlConf,
                              ServerConfiguration bkConf,
                              int numBookies,
                              boolean shouldStartZK,
                              String zkServers,
                              int zkPort,
                              boolean shouldStartProxy,
                              int proxyPort) throws Exception {
    this.dlConf = dlConf;
    if (shouldStartZK) {
        File zkTmpDir = IOUtils.createTempDir("zookeeper", "distrlog");
        tmpDirs.add(zkTmpDir);
        if (0 == zkPort) {
            Pair<ZooKeeperServerShim, Integer> serverAndPort = LocalDLMEmulator.runZookeeperOnAnyPort(zkTmpDir);
            this.zks = serverAndPort.getLeft();
            zkPort = serverAndPort.getRight();
        } else {
            this.zks = LocalBookKeeper.runZookeeper(1000, zkPort, zkTmpDir);
        }
    } else {
        this.zks = null;
    }
    this.dlmEmulator = LocalDLMEmulator.newBuilder()
            .numBookies(numBookies)
            .zkHost(zkServers)
            .zkPort(zkPort)
            .serverConf(bkConf)
            .shouldStartZK(false)
            .build();
    this.shouldStartProxy = shouldStartProxy;
    this.proxyPort = proxyPort;
}
 

private DistributedLogCluster(DistributedLogConfiguration dlConf,
                              ServerConfiguration bkConf,
                              int numBookies,
                              boolean shouldStartZK,
                              String zkServers,
                              int zkPort,
                              boolean shouldStartProxy,
                              int proxyPort,
                              boolean thriftmux) throws Exception {
    this.dlConf = dlConf;
    if (shouldStartZK) {
        File zkTmpDir = IOUtils.createTempDir("zookeeper", "distrlog");
        tmpDirs.add(zkTmpDir);
        if (0 == zkPort) {
            Pair<ZooKeeperServerShim, Integer> serverAndPort = LocalDLMEmulator.runZookeeperOnAnyPort(zkTmpDir);
            this.zks = serverAndPort.getLeft();
            zkPort = serverAndPort.getRight();
        } else {
            this.zks = LocalBookKeeper.runZookeeper(1000, zkPort, zkTmpDir);
        }
    } else {
        this.zks = null;
    }
    this.dlmEmulator = LocalDLMEmulator.newBuilder()
            .numBookies(numBookies)
            .zkHost(zkServers)
            .zkPort(zkPort)
            .serverConf(bkConf)
            .shouldStartZK(false)
            .build();
    this.shouldStartProxy = shouldStartProxy;
    this.proxyPort = proxyPort;
    this.thriftmux = thriftmux;
}
 

public void initialize() throws IOException {
    System.setProperty("zookeeper.4lw.commands.whitelist", "*"); // As of ZooKeeper 3.5 this is needed to not break start()
    if (this.tmpDir.compareAndSet(null, IOUtils.createTempDir("zookeeper", "inproc"))) {
        this.tmpDir.get().deleteOnExit();
    }
    if (secureZK) {
        ZKTLSUtils.setSecureZKServerProperties(this.keyStore, this.keyStorePasswordPath, this.trustStore, this.keyStorePasswordPath);
    }
}
 

@BeforeClass
public static void setupZooKeeper() throws Exception {
    zkDir = IOUtils.createTempDir("zookeeper", ZooKeeperClusterTestCase.class.getName());
    Pair<ZooKeeperServerShim, Integer> serverAndPort = LocalDLMEmulator.runZookeeperOnAnyPort(zkDir);
    zks = serverAndPort.getLeft();
    zkPort = serverAndPort.getRight();
    zkServers = "127.0.0.1:" + zkPort;
}
 

@Override
protected int runCmd(CommandLine cmdline) throws Exception {
    String[] args = cmdline.getArgs();
    if (args.length <= 0) {
        System.err.println("No distributedlog uri specified.");
        printUsage();
        return -1;
    }
    boolean force = cmdline.hasOption("f");
    URI uri = URI.create(args[0]);
    // resolving the uri to see if there is another bindings in this uri.
    ZooKeeperClient zkc = ZooKeeperClientBuilder.newBuilder().uri(uri)
            .sessionTimeoutMs(10000).build();
    BKDLConfig bkdlConfig;
    try {
        bkdlConfig = BKDLConfig.resolveDLConfig(zkc, uri);
    } catch (IOException ie) {
        bkdlConfig = null;
    }
    if (null == bkdlConfig) {
        System.out.println("No bookkeeper is bound to " + uri);
        return 0;
    } else {
        System.out.println("There is bookkeeper bound to " + uri + " : ");
        System.out.println("");
        System.out.println(bkdlConfig.toString());
        System.out.println("");
        if (!force && !IOUtils.confirmPrompt("Do you want to unbind " + uri + " :\n")) {
            return 0;
        }
    }
    DLMetadata.unbind(uri);
    System.out.println("Unbound on " + uri + ".");
    return 0;
}
 

public static void main(String[] args) throws Exception {
    try {
        if (args.length < 1) {
            System.out.println("Usage: LocalDLEmulator <zk_port>");
            System.exit(-1);
        }

        final int zkPort = Integer.parseInt(args[0]);
        final File zkDir = IOUtils.createTempDir("distrlog", "zookeeper");
        final LocalDLMEmulator localDlm = LocalDLMEmulator.newBuilder()
            .zkPort(zkPort)
            .build();

        Runtime.getRuntime().addShutdownHook(new Thread() {
            @Override
            public void run() {
                try {
                    localDlm.teardown();
                    FileUtils.forceDeleteOnExit(zkDir);
                    System.out.println("ByeBye!");
                } catch (Exception e) {
                    // do nothing
                }
            }
        });
        localDlm.start();

        System.out.println(String.format(
            "DistributedLog Sandbox is running now. You could access distributedlog://%s:%s",
            DEFAULT_ZK_HOST,
            zkPort));
    } catch (Exception ex) {
        System.out.println("Exception occurred running emulator " + ex);
    }
}
 

private int truncateStream(final Namespace namespace, String streamName, DLSN dlsn) throws Exception {
    DistributedLogManager dlm = namespace.openLog(streamName);
    try {
        long totalRecords = dlm.getLogRecordCount();
        long recordsAfterTruncate = FutureUtils.result(dlm.getLogRecordCountAsync(dlsn));
        long recordsToTruncate = totalRecords - recordsAfterTruncate;
        if (!getForce() && !IOUtils.confirmPrompt("Do you want to truncate "
                + streamName + " at dlsn " + dlsn + " (" + recordsToTruncate + " records)?")) {
            return 0;
        } else {
            AsyncLogWriter writer = dlm.startAsyncLogSegmentNonPartitioned();
            try {
                if (!FutureUtils.result(writer.truncate(dlsn))) {
                    System.out.println("Failed to truncate.");
                }
                return 0;
            } finally {
                Utils.close(writer);
            }
        }
    } catch (Exception ex) {
        System.err.println("Failed to truncate " + ex);
        return 1;
    } finally {
        dlm.close();
    }
}
 

protected void repairLogSegment(BookKeeperAdmin bkAdmin,
                                MetadataUpdater metadataUpdater,
                                LogSegmentMetadata segment) throws Exception {
    if (segment.isInProgress()) {
        System.out.println("Skip inprogress log segment " + segment);
        return;
    }
    LedgerHandle lh = bkAdmin.openLedger(segment.getLogSegmentId());
    long lac = lh.getLastAddConfirmed();
    Enumeration<LedgerEntry> entries = lh.readEntries(lac, lac);
    if (!entries.hasMoreElements()) {
        throw new IOException("Entry " + lac + " isn't found for " + segment);
    }
    LedgerEntry lastEntry = entries.nextElement();
    Entry.Reader reader = Entry.newBuilder()
            .setLogSegmentInfo(segment.getLogSegmentSequenceNumber(), segment.getStartSequenceId())
            .setEntryId(lastEntry.getEntryId())
            .setEnvelopeEntry(LogSegmentMetadata.supportsEnvelopedEntries(segment.getVersion()))
            .setEntry(lastEntry.getEntryBuffer())
            .buildReader();
    lastEntry.getEntryBuffer().release();
    LogRecordWithDLSN record = reader.nextRecord();
    LogRecordWithDLSN lastRecord = null;
    while (null != record) {
        lastRecord = record;
        record = reader.nextRecord();
    }
    if (null == lastRecord) {
        throw new IOException("No record found in entry " + lac + " for " + segment);
    }
    System.out.println("Updating last record for " + segment + " to " + lastRecord);
    if (!IOUtils.confirmPrompt("Do you want to make this change (Y/N): ")) {
        return;
    }
    metadataUpdater.updateLastRecord(segment, lastRecord);
}
 

protected int inspectAndRepair(List<LogSegmentMetadata> segments) throws Exception {
    LogSegmentMetadataStore metadataStore = getLogSegmentMetadataStore();
    ZooKeeperClient zkc = getZooKeeperClient();
    BKDLConfig bkdlConfig = BKDLConfig.resolveDLConfig(zkc, getUri());
    BKDLConfig.propagateConfiguration(bkdlConfig, getConf());
    BookKeeperClient bkc = BookKeeperClientBuilder.newBuilder()
            .dlConfig(getConf())
            .zkServers(bkdlConfig.getBkZkServersForReader())
            .ledgersPath(bkdlConfig.getBkLedgersPath())
            .name("dlog")
            .build();
    try {
        List<LogSegmentMetadata> segmentsToRepair = inspectLogSegments(bkc, segments);
        if (segmentsToRepair.isEmpty()) {
            System.out.println("The stream is good. No log segments to repair.");
            return 0;
        }
        System.out.println(segmentsToRepair.size() + " segments to repair : ");
        System.out.println(segmentsToRepair);
        System.out.println();
        if (!IOUtils.confirmPrompt("Do you want to repair them (Y/N): ")) {
            return 0;
        }
        repairLogSegments(metadataStore, bkc, segmentsToRepair);
        return 0;
    } finally {
        bkc.close();
    }
}
 

public static void checkAndRepairDLNamespace(final URI uri,
                                             final Namespace namespace,
                                             final MetadataUpdater metadataUpdater,
                                             final OrderedScheduler scheduler,
                                             final boolean verbose,
                                             final boolean interactive,
                                             final int concurrency) throws Exception {
    checkArgument(concurrency > 0, "Invalid concurrency " + concurrency + " found.");
    // 0. getting streams under a given uri.
    Iterator<String> streamsIter = namespace.getLogs();
    List<String> streams = Lists.newArrayList();
    while (streamsIter.hasNext()) {
        streams.add(streamsIter.next());
    }
    if (verbose) {
        System.out.println("- 0. checking streams under " + uri);
    }
    if (streams.size() == 0) {
        System.out.println("+ 0. nothing to check. quit.");
        return;
    }
    Map<String, StreamCandidate> streamCandidates =
            checkStreams(namespace, streams, scheduler, concurrency);
    if (verbose) {
        System.out.println("+ 0. " + streamCandidates.size() + " corrupted streams found.");
    }
    if (interactive && !IOUtils.confirmPrompt("Do you want to fix all "
            + streamCandidates.size() + " corrupted streams (Y/N) : ")) {
        return;
    }
    if (verbose) {
        System.out.println("- 1. repairing " + streamCandidates.size() + " corrupted streams.");
    }
    for (StreamCandidate candidate : streamCandidates.values()) {
        if (!repairStream(metadataUpdater, candidate, verbose, interactive)) {
            if (verbose) {
                System.out.println("* 1. aborted repairing corrupted streams.");
            }
            return;
        }
    }
    if (verbose) {
        System.out.println("+ 1. repaired " + streamCandidates.size() + " corrupted streams.");
    }
}
 

@Override
protected int runCmd() throws Exception {
    String rootPath = getUri().getPath() + "/" + allocationPoolPath;
    final ScheduledExecutorService allocationExecutor = Executors.newSingleThreadScheduledExecutor();
    ExecutorService executorService = Executors.newFixedThreadPool(concurrency);
    checkArgument(getNamespace() instanceof BKDistributedLogNamespace);
    BKDistributedLogNamespace bkns = (BKDistributedLogNamespace) getNamespace();
    final ZooKeeperClient zkc = ((BKNamespaceDriver) bkns.getNamespaceDriver()).getWriterZKC();
    final BookKeeperClient bkc = ((BKNamespaceDriver) bkns.getNamespaceDriver()).getReaderBKC();
    try {
        List<String> pools = zkc.get().getChildren(rootPath, false);
        final LinkedBlockingQueue<String> poolsToDelete = new LinkedBlockingQueue<String>();
        if (getForce() || IOUtils.confirmPrompt("Are you sure you want to delete allocator pools : " + pools)) {
            for (String pool : pools) {
                poolsToDelete.add(rootPath + "/" + pool);
            }
            final CountDownLatch doneLatch = new CountDownLatch(concurrency);
            for (int i = 0; i < concurrency; i++) {
                final int tid = i;
                executorService.submit(new Runnable() {
                    @Override
                    public void run() {
                        while (!poolsToDelete.isEmpty()) {
                            String poolPath = poolsToDelete.poll();
                            if (null == poolPath) {
                                break;
                            }
                            try {
                                LedgerAllocator allocator =
                                        LedgerAllocatorUtils.createLedgerAllocatorPool(poolPath, 0, getConf(),
                                                zkc, bkc,
                                                allocationExecutor);
                                allocator.delete();
                                System.out.println("Deleted allocator pool : " + poolPath + " .");
                            } catch (IOException ioe) {
                                System.err.println("Failed to delete allocator pool "
                                        + poolPath + " : " + ioe.getMessage());
                            }
                        }
                        doneLatch.countDown();
                        System.out.println("Thread " + tid + " is done.");
                    }
                });
            }
            doneLatch.await();
        }
    } finally {
        executorService.shutdown();
        allocationExecutor.shutdown();
    }
    return 0;
}
 

private void inspectStreams(final SortedMap<String,
        List<Pair<LogSegmentMetadata, List<String>>>> corruptedCandidates) throws Exception {
    Iterator<String> streamCollection = getNamespace().getLogs();
    final List<String> streams = new ArrayList<String>();
    while (streamCollection.hasNext()) {
        String s = streamCollection.next();
        if (null != streamPrefix) {
            if (s.startsWith(streamPrefix)) {
                streams.add(s);
            }
        } else {
            streams.add(s);
        }
    }
    if (0 == streams.size()) {
        return;
    }
    println("Streams : " + streams);
    if (!getForce() && !IOUtils.confirmPrompt("Are you sure you want to inspect "
            + streams.size() + " streams")) {
        return;
    }
    numThreads = Math.min(streams.size(), numThreads);
    final int numStreamsPerThreads = streams.size() / numThreads;
    Thread[] threads = new Thread[numThreads];
    for (int i = 0; i < numThreads; i++) {
        final int tid = i;
        threads[i] = new Thread("Inspect-" + i) {
            @Override
            public void run() {
                try {
                    inspectStreams(streams, tid, numStreamsPerThreads, corruptedCandidates);
                    System.out.println("Thread " + tid + " finished.");
                } catch (Exception e) {
                    System.err.println("Thread " + tid + " quits with exception : " + e.getMessage());
                }
            }
        };
        threads[i].start();
    }
    for (int i = 0; i < numThreads; i++) {
        threads[i].join();
    }
}