Java Code Examples for com.twitter.util.Future#collect()

private Future<List<Void>> closeStreams(Set<Stream> streamsToClose, Optional<RateLimiter> rateLimiter) {
    if (streamsToClose.isEmpty()) {
        logger.info("No streams to close.");
        List<Void> emptyList = new ArrayList<Void>();
        return Future.value(emptyList);
    }
    List<Future<Void>> futures = new ArrayList<Future<Void>>(streamsToClose.size());
    for (Stream stream : streamsToClose) {
        if (rateLimiter.isPresent()) {
            rateLimiter.get().acquire();
        }
        futures.add(stream.requestClose("Close Streams"));
    }
    return Future.collect(futures);
}
 

static Future<List<Versioned<byte[]>>> checkLogMetadataPaths(ZooKeeper zk,
                                                             String logRootPath,
                                                             boolean ownAllocator) {
    // Note re. persistent lock state initialization: the read lock persistent state (path) is
    // initialized here but only used in the read handler. The reason is its more convenient and
    // less error prone to manage all stream structure in one place.
    final String logRootParentPath = new File(logRootPath).getParent();
    final String logSegmentsPath = logRootPath + LOGSEGMENTS_PATH;
    final String maxTxIdPath = logRootPath + MAX_TXID_PATH;
    final String lockPath = logRootPath + LOCK_PATH;
    final String readLockPath = logRootPath + READ_LOCK_PATH;
    final String versionPath = logRootPath + VERSION_PATH;
    final String allocationPath = logRootPath + ALLOCATION_PATH;

    int numPaths = ownAllocator ? MetadataIndex.ALLOCATION + 1 : MetadataIndex.LOGSEGMENTS + 1;
    List<Future<Versioned<byte[]>>> checkFutures = Lists.newArrayListWithExpectedSize(numPaths);
    checkFutures.add(Utils.zkGetData(zk, logRootParentPath, false));
    checkFutures.add(Utils.zkGetData(zk, logRootPath, false));
    checkFutures.add(Utils.zkGetData(zk, maxTxIdPath, false));
    checkFutures.add(Utils.zkGetData(zk, versionPath, false));
    checkFutures.add(Utils.zkGetData(zk, lockPath, false));
    checkFutures.add(Utils.zkGetData(zk, readLockPath, false));
    checkFutures.add(Utils.zkGetData(zk, logSegmentsPath, false));
    if (ownAllocator) {
        checkFutures.add(Utils.zkGetData(zk, allocationPath, false));
    }

    return Future.collect(checkFutures);
}
 

private Future<List<Set<String>>> retrieveLogs() {
    Collection<SubNamespace> subNss = subNamespaces.values();
    List<Future<Set<String>>> logsList = Lists.newArrayListWithExpectedSize(subNss.size());
    for (SubNamespace subNs : subNss) {
        logsList.add(subNs.getLogs());
    }
    return Future.collect(logsList);
}
 

private Future<List<Void>> closeStreams(Set<Stream> streamsToClose, Optional<RateLimiter> rateLimiter) {
    if (streamsToClose.isEmpty()) {
        logger.info("No streams to close.");
        List<Void> emptyList = new ArrayList<Void>();
        return Future.value(emptyList);
    }
    List<Future<Void>> futures = new ArrayList<Future<Void>>(streamsToClose.size());
    for (Stream stream : streamsToClose) {
        if (rateLimiter.isPresent()) {
            rateLimiter.get().acquire();
        }
        futures.add(stream.requestClose("Close Streams"));
    }
    return Future.collect(futures);
}
 

@Override
public Response delete(LindenDeleteRequest request) throws IOException {
  List<Future<BoxedUnit>> futures = new ArrayList<>();
  List<String> hosts = new ArrayList<>();
  final StringBuilder errorInfo = new StringBuilder();
  Set<Integer> routeIds = null;
  if (request.isSetRouteParam()) {
    routeIds = new HashSet<>();
    for (final ShardRouteParam routeParam : request.getRouteParam().getShardParams()) {
      routeIds.add(routeParam.getShardId());
    }
  }
  for (final Map.Entry<Integer, ShardClient> entry : clients.entrySet()) {
    if (routeIds != null && !routeIds.contains(entry.getKey())) {
      continue;
    }
    if (entry.getValue().isAvailable()) {
      List<Map.Entry<String, Future<Response>>> hostFuturePairs = entry.getValue().delete(request);
      for (final Map.Entry<String, Future<Response>> hostFuturePair : hostFuturePairs) {
        hosts.add(hostFuturePair.getKey());
        futures.add(hostFuturePair.getValue().transformedBy(new FutureTransformer<Response, BoxedUnit>() {
          @Override
          public BoxedUnit map(Response response) {
            if (!response.isSuccess()) {
              synchronized (errorInfo) {
                LOGGER.error("Shard [{}] host [{}] failed to get delete response : {}",
                             entry.getKey(), hostFuturePair.getKey(), response.getError());
                errorInfo
                    .append("Shard " + entry.getKey() + " host " + hostFuturePair.getKey() + ":" + response.getError()
                            + ";");

              }
            }
            return BoxedUnit.UNIT;
          }

          @Override
          public BoxedUnit handle(Throwable t) {
            LOGGER.error("Shard [{}] host [{}] failed to get delete response : {}",
                         entry.getKey(), hostFuturePair.getKey(), Throwables.getStackTraceAsString(t));
            errorInfo
                .append("Shard " + entry.getKey() + " host " + hostFuturePair.getKey() + ":" + Throwables
                    .getStackTraceAsString(t) + ";");
            return BoxedUnit.UNIT;
          }
        }));
      }
    }
  }

  Future<List<BoxedUnit>> collected = Future.collect(futures);
  try {
    if (clusterFutureAwaitTimeout == 0) {
      Await.result(collected);
    } else {
      Await.result(collected, Duration.apply(clusterFutureAwaitTimeout, TimeUnit.MILLISECONDS));
    }
    if (errorInfo.length() > 0) {
      return ResponseUtils.buildFailedResponse("Delete failed: " + errorInfo.toString());
    }
    return ResponseUtils.SUCCESS;
  } catch (Exception e) {
    LOGGER.error("Failed to get all delete responses, exception: {}", Throwables.getStackTraceAsString(e));
    LOGGER.error(getHostFutureInfo(hosts, futures));
    return ResponseUtils.buildFailedResponse(e);
  }
}
 

@Override
public Response index(String content) throws IOException {
  List<Future<BoxedUnit>> futures = new ArrayList<>();
  List<String> hosts = new ArrayList<>();
  final StringBuilder errorInfo = new StringBuilder();
  LindenIndexRequest indexRequest = LindenIndexRequestParser.parse(lindenConfig.getSchema(), content);
  for (final Map.Entry<Integer, ShardClient> entry : clients.entrySet()) {
    ShardClient shardClient = entry.getValue();
    if (shardClient.isAvailable()) {
      if (shardingStrategy
          .accept(indexRequest.getId(), indexRequest.getRouteParam(), shardClient.getShardId())) {
        final List<Map.Entry<String, Future<Response>>> hostFuturePairs = shardClient.index(content);
        for (final Map.Entry<String, Future<Response>> hostFuturePair : hostFuturePairs) {
          hosts.add(hostFuturePair.getKey());
          futures.add(hostFuturePair.getValue().transformedBy(new FutureTransformer<Response, BoxedUnit>() {
            @Override
            public BoxedUnit map(Response response) {
              if (!response.isSuccess()) {
                LOGGER.error("Shard [{}] host [{}] failed to get index response : {}",
                             entry.getKey(), hostFuturePair.getKey(), response.getError());
                synchronized (errorInfo) {
                  errorInfo.append(
                      "Shard " + entry.getKey() + " host " + hostFuturePair.getKey() + ":" + response.getError()
                      + ";");
                }
              }
              return BoxedUnit.UNIT;
            }

            @Override
            public BoxedUnit handle(Throwable t) {
              LOGGER.error("Shard [{}] host [{}] failed to get index response : {}",
                           entry.getKey(), hostFuturePair.getKey(), Throwables.getStackTraceAsString(t));
              synchronized (errorInfo) {
                errorInfo.append("Shard " + entry.getKey() + " host " + hostFuturePair.getKey() + ":" + Throwables
                    .getStackTraceAsString(t) + ";");
              }
              return BoxedUnit.UNIT;
            }
          }));
        }
      }
    }
  }
  try {
    Future<List<BoxedUnit>> collected = Future.collect(futures);
    if (clusterFutureAwaitTimeout == 0) {
      Await.result(collected);
    } else {
      Await.result(collected, Duration.apply(clusterFutureAwaitTimeout, TimeUnit.MILLISECONDS));
    }
    if (errorInfo.length() > 0) {
      return ResponseUtils.buildFailedResponse("Index failed: " + errorInfo.toString());
    }
    return ResponseUtils.SUCCESS;
  } catch (Exception e) {
    LOGGER.error("Handle request failed, content : {} - {}", content, Throwables.getStackTraceAsString(e));
    LOGGER.error(getHostFutureInfo(hosts, futures));
    return ResponseUtils.buildFailedResponse(e);
  }
}
 

@Override
public Response executeCommand(final String command) throws IOException {
  LOGGER.info("Receive cluster command {}", command);
  List<Future<BoxedUnit>> futures = new ArrayList<>();
  List<String> hosts = new ArrayList<>();
  final StringBuilder errorInfo = new StringBuilder();
  JSONObject jsonCmd = JSONObject.parseObject(command);
  Integer shardId = jsonCmd.getInteger(SHARD);
  for (final Map.Entry<Integer, ShardClient> entry : clients.entrySet()) {
    ShardClient shardClient = entry.getValue();
    if (shardId != null && shardId != shardClient.getShardId()) {
      continue;
    }
    if (shardClient.isAvailable()) {
      final List<Map.Entry<String, Future<Response>>> hostFuturePairs = shardClient.executeCommand(jsonCmd);
      for (final Map.Entry<String, Future<Response>> hostFuturePair : hostFuturePairs) {
        hosts.add(hostFuturePair.getKey());
        futures.add(hostFuturePair.getValue().transformedBy(new FutureTransformer<Response, BoxedUnit>() {
          @Override
          public BoxedUnit map(Response response) {
            if (!response.isSuccess()) {
              LOGGER.error("Shard [{}] host [{}] failed to execute command {} error: {}",
                           entry.getKey(), hostFuturePair.getKey(), command, response.getError());
              synchronized (errorInfo) {
                errorInfo.append(
                    "Shard " + entry.getKey() + " host " + hostFuturePair.getKey() + ":" + response.getError() + ";");
              }
            }
            return BoxedUnit.UNIT;
          }

          @Override
          public BoxedUnit handle(Throwable t) {
            LOGGER.error("Shard [{}] host [{}] failed to execute command {} throwable: {}\"",
                         entry.getKey(), hostFuturePair.getKey(), command, Throwables.getStackTraceAsString(t));
            synchronized (errorInfo) {
              errorInfo.append("Shard " + entry.getKey() + " host " + hostFuturePair.getKey() + ":" + Throwables
                  .getStackTraceAsString(t) + ";");
            }
            return BoxedUnit.UNIT;
          }
        }));
      }
    }
  }
  try {
    Future<List<BoxedUnit>> collected = Future.collect(futures);
    if (clusterFutureAwaitTimeout == 0) {
      Await.result(collected);
    } else {
      // executeCommand may take a very long time, so set timeout to 30min specially
      Await.result(collected, Duration.apply(30, TimeUnit.MINUTES));
    }
    if (errorInfo.length() > 0) {
      return ResponseUtils.buildFailedResponse("command " + command + " failed: " + errorInfo.toString());
    }
    LOGGER.error("Cluster command {} succeeded", command);
    return ResponseUtils.SUCCESS;
  } catch (Exception e) {
    LOGGER.error("Cluster command {} failed {}", command, Throwables.getStackTraceAsString(e));
    LOGGER.error(getHostFutureInfo(hosts, futures));
    return ResponseUtils.buildFailedResponse(e);
  }
}
 

@Override
public LindenResult search(final LindenSearchRequest request) throws IOException {
  final List<LindenResult> resultList = new ArrayList<>();
  final List<Future<BoxedUnit>> futures = new ArrayList<>();

  List<String> indexNames;
  // Only INDEX_NAME division type supports specified index name request
  if (multiIndexStrategy instanceof TimeLimitMultiIndexStrategy
      || multiIndexStrategy instanceof DocNumLimitMultiIndexStrategy) {
    indexNames = new ArrayList<>(lindenCoreMap.keySet());
  } else {
    if (request.getIndexNames() == null || (request.getIndexNamesSize() == 1 && request.getIndexNames().get(0)
        .equals(LINDEN))) {
      indexNames = new ArrayList<>(lindenCoreMap.keySet());
    } else {
      indexNames = request.getIndexNames();
      for (int i = 0; i < indexNames.size(); ++i) {
        indexNames.set(i, MultiIndexStrategy.MULTI_INDEX_PREFIX_NAME + indexNames.get(i));
      }
    }
  }

  for (final String indexName : indexNames) {
    final LindenCore core = lindenCoreMap.get(indexName);
    if (core != null) {
      futures
          .add(Future.value(core.search(request)).transformedBy(new FutureTransformer<LindenResult, BoxedUnit>() {
            @Override
            public BoxedUnit map(LindenResult lindenResult) {
              synchronized (resultList) {
                resultList.add(lindenResult);
              }
              return BoxedUnit.UNIT;
            }

            @Override
            public BoxedUnit handle(Throwable t) {
              LOGGER.info("Index {} search error: {}", indexName,
                          Throwables.getStackTraceAsString(t));
              return BoxedUnit.UNIT;
            }
          }));
    } else {
      LOGGER.error("Index {} doesn't exist.", indexName);
    }
  }
  Future<List<BoxedUnit>> collected = Future.collect(futures);
  try {
    collected.apply(Duration.apply(DEFAULT_SEARCH_TIMEOUT, TimeUnit.MILLISECONDS));
  } catch (Exception e) {
    LOGGER.error("Multi-index search error: {}", Throwables.getStackTraceAsString(e));
  }
  return ResultMerger.merge(request, resultList);
}