Java Code Examples for org.apache.distributedlog.api.namespace.Namespace#getLogs()

protected void printStreams(Namespace namespace) throws Exception {
    Iterator<String> streams = namespace.getLogs();
    System.out.println("Streams under " + getUri() + " : ");
    System.out.println("--------------------------------");
    while (streams.hasNext()) {
        String streamName = streams.next();
        System.out.println(streamName);
        if (!printMetadata) {
            continue;
        }
        MetadataAccessor accessor =
                namespace.getNamespaceDriver().getMetadataAccessor(streamName);
        byte[] metadata = accessor.getMetadata();
        if (null == metadata || metadata.length == 0) {
            continue;
        }
        if (printHex) {
            System.out.println(Hex.encodeHexString(metadata));
        } else {
            System.out.println(new String(metadata, UTF_8));
        }
        System.out.println("");
    }
    System.out.println("--------------------------------");
}
 

private void collectLedgersFromDL(final URI uri,
                                  final Namespace namespace,
                                  final Set<Long> ledgers) throws IOException {
    logger.info("Enumerating {} to collect streams.", uri);
    Iterator<String> streams = namespace.getLogs();
    final LinkedBlockingQueue<String> streamQueue = new LinkedBlockingQueue<String>();
    while (streams.hasNext()) {
        streamQueue.add(streams.next());
    }

    logger.info("Collected {} streams from uri {} : {}",
                new Object[] { streamQueue.size(), uri, streams });

    executeAction(streamQueue, 10, new Action<String>() {
        @Override
        public void execute(String stream) throws IOException {
            collectLedgersFromStream(namespace, stream, ledgers);
        }
    });
}
 

private Map<String, Long> calculateStreamSpaceUsage(
        final URI uri, final Namespace namespace)
    throws IOException {
    Iterator<String> streams = namespace.getLogs();
    final LinkedBlockingQueue<String> streamQueue = new LinkedBlockingQueue<String>();
    while (streams.hasNext()) {
        streamQueue.add(streams.next());
    }

    final Map<String, Long> streamSpaceUsageMap =
            new ConcurrentSkipListMap<String, Long>();
    final AtomicInteger numStreamsCollected = new AtomicInteger(0);

    executeAction(streamQueue, 10, new Action<String>() {
        @Override
        public void execute(String stream) throws IOException {
            streamSpaceUsageMap.put(stream,
                    calculateStreamSpaceUsage(namespace, stream));
            if (numStreamsCollected.incrementAndGet() % 1000 == 0) {
                logger.info("Calculated {} streams from uri {}.", numStreamsCollected.get(), uri);
            }
        }
    });

    return streamSpaceUsageMap;
}
 

@Test(timeout = 60000)
public void testCheckLogExists() throws Exception {
    String name = "distrlog-check-log-exists";
    DistributedLogManager dlm = createNewDLM(conf, name);

    long txid = 1;
    LogWriter writer = dlm.startLogSegmentNonPartitioned();
    for (long j = 1; j <= DEFAULT_SEGMENT_SIZE / 2; j++) {
        writer.write(DLMTestUtil.getLogRecordInstance(txid++));
    }
    writer.flush();
    writer.commit();
    writer.close();
    dlm.close();

    URI uri = createDLMURI("/" + name);
    Namespace namespace = NamespaceBuilder.newBuilder()
            .conf(conf).uri(uri).build();
    assertTrue(namespace.logExists(name));
    assertFalse(namespace.logExists("non-existent-log"));
    URI nonExistentUri = createDLMURI("/" + "non-existent-ns");
    Namespace nonExistentNS = NamespaceBuilder.newBuilder()
            .conf(conf).uri(nonExistentUri).build();
    assertFalse(nonExistentNS.logExists(name));

    int logCount = 0;
    Iterator<String> logIter = namespace.getLogs();
    while (logIter.hasNext()) {
        String log = logIter.next();
        logCount++;
        assertEquals(name, log);
    }
    assertEquals(1, logCount);

    namespace.close();
}
 

@Override
public void dispose(String oldestCheckpointId, boolean deleteAll)
    throws StatefulStorageException {

  // Currently dlog doesn't support recursive deletion. so we have to fetch all the checkpoints
  // and delete individual checkpoints.
  // TODO (sijie): replace the logic here once distributedlog supports recursive deletion.

  String topologyCheckpointRoot = getTopologyCheckpointRoot(topologyName);
  URI topologyUri = URI.create(checkpointNamespaceUriStr + topologyCheckpointRoot);
  // get checkpoints
  Namespace topologyNs = null;
  Iterator<String> checkpoints;
  try {
    topologyNs = initializeNamespace(topologyUri);
    checkpoints = topologyNs.getLogs();
  } catch (IOException ioe) {
    throw new StatefulStorageException("Failed to open topology namespace", ioe);
  } finally {
    if (null != topologyNs) {
      topologyNs.close();
    }
  }

  while (checkpoints.hasNext()) {
    String checkpointId = checkpoints.next();
    if (deleteAll || checkpointId.compareTo(oldestCheckpointId) < 0) {
      URI checkpointUri =
          URI.create(checkpointNamespaceUriStr + topologyCheckpointRoot + "/" + checkpointId);
      try {
        deleteCheckpoint(checkpointUri);
      } catch (IOException e) {
        throw new StatefulStorageException("Failed to remove checkpoint "
            + checkpointId + " for topology " + topologyName, e);
      }
    }
  }
}
 

private void deleteCheckpoint(URI checkpointUri) throws IOException {
  Namespace checkpointNs = initializeNamespace(checkpointUri);
  try {
    Iterator<String> checkpoints = checkpointNs.getLogs();
    while (checkpoints.hasNext()) {
      String checkpoint = checkpoints.next();
      checkpointNs.deleteLog(checkpoint);
    }
  } finally {
    checkpointNs.close();
  }
}
 

public static void checkAndRepairDLNamespace(final URI uri,
                                             final Namespace namespace,
                                             final MetadataUpdater metadataUpdater,
                                             final OrderedScheduler scheduler,
                                             final boolean verbose,
                                             final boolean interactive,
                                             final int concurrency) throws Exception {
    checkArgument(concurrency > 0, "Invalid concurrency " + concurrency + " found.");
    // 0. getting streams under a given uri.
    Iterator<String> streamsIter = namespace.getLogs();
    List<String> streams = Lists.newArrayList();
    while (streamsIter.hasNext()) {
        streams.add(streamsIter.next());
    }
    if (verbose) {
        System.out.println("- 0. checking streams under " + uri);
    }
    if (streams.size() == 0) {
        System.out.println("+ 0. nothing to check. quit.");
        return;
    }
    Map<String, StreamCandidate> streamCandidates =
            checkStreams(namespace, streams, scheduler, concurrency);
    if (verbose) {
        System.out.println("+ 0. " + streamCandidates.size() + " corrupted streams found.");
    }
    if (interactive && !IOUtils.confirmPrompt("Do you want to fix all "
            + streamCandidates.size() + " corrupted streams (Y/N) : ")) {
        return;
    }
    if (verbose) {
        System.out.println("- 1. repairing " + streamCandidates.size() + " corrupted streams.");
    }
    for (StreamCandidate candidate : streamCandidates.values()) {
        if (!repairStream(metadataUpdater, candidate, verbose, interactive)) {
            if (verbose) {
                System.out.println("* 1. aborted repairing corrupted streams.");
            }
            return;
        }
    }
    if (verbose) {
        System.out.println("+ 1. repaired " + streamCandidates.size() + " corrupted streams.");
    }
}
 

private int truncateStreams(final Namespace namespace) throws Exception {
    Iterator<String> streamCollection = namespace.getLogs();
    final List<String> streams = new ArrayList<String>();
    while (streamCollection.hasNext()) {
        String s = streamCollection.next();
        if (null != streamPrefix) {
            if (s.startsWith(streamPrefix)) {
                streams.add(s);
            }
        } else {
            streams.add(s);
        }
    }
    if (0 == streams.size()) {
        return 0;
    }
    System.out.println("Streams : " + streams);
    if (!getForce() && !IOUtils.confirmPrompt("Do you want to truncate " + streams.size() + " streams ?")) {
        return 0;
    }
    numThreads = Math.min(streams.size(), numThreads);
    final int numStreamsPerThreads = streams.size() / numThreads + 1;
    Thread[] threads = new Thread[numThreads];
    for (int i = 0; i < numThreads; i++) {
        final int tid = i;
        threads[i] = new Thread("Truncate-" + i) {
            @Override
            public void run() {
                try {
                    truncateStreams(namespace, streams, tid, numStreamsPerThreads);
                    System.out.println("Thread " + tid + " finished.");
                } catch (IOException e) {
                    System.err.println("Thread " + tid + " quits with exception : " + e.getMessage());
                }
            }
        };
        threads[i].start();
    }
    for (int i = 0; i < numThreads; i++) {
        threads[i].join();
    }
    return 0;
}
 

private int deleteSubscriber(final Namespace namespace) throws Exception {
    Iterator<String> streamCollection = namespace.getLogs();
    final List<String> streams = new ArrayList<String>();
    while (streamCollection.hasNext()) {
        String s = streamCollection.next();
        if (null != streamPrefix) {
            if (s.startsWith(streamPrefix)) {
                streams.add(s);
            }
        } else {
            streams.add(s);
        }
    }
    if (0 == streams.size()) {
        return 0;
    }
    System.out.println("Streams : " + streams);
    if (!getForce() && !IOUtils.confirmPrompt("Do you want to delete subscriber "
        + subscriberId + " for " + streams.size() + " streams ?")) {
        return 0;
    }
    numThreads = Math.min(streams.size(), numThreads);
    final int numStreamsPerThreads = streams.size() / numThreads + 1;
    Thread[] threads = new Thread[numThreads];
    for (int i = 0; i < numThreads; i++) {
        final int tid = i;
        threads[i] = new Thread("RemoveSubscriberThread-" + i) {
            @Override
            public void run() {
                try {
                    deleteSubscriber(namespace, streams, tid, numStreamsPerThreads);
                    System.out.println("Thread " + tid + " finished.");
                } catch (Exception e) {
                    System.err.println("Thread " + tid + " quits with exception : " + e.getMessage());
                }
            }
        };
        threads[i].start();
    }
    for (int i = 0; i < numThreads; i++) {
        threads[i].join();
    }
    return 0;
}