Java Code Examples for com.google.common.collect.ImmutableMap#Entry

/** Resolve the map of names to SourcePaths to a map of names to CxxSource objects. */
private static ImmutableMap<String, CxxSource> resolveCxxSources(
    ImmutableMap<String, SourceWithFlags> sources) {

  ImmutableMap.Builder<String, CxxSource> cxxSources = ImmutableMap.builder();

  // For each entry in the input C/C++ source, build a CxxSource object to wrap
  // it's name, input path, and output object file path.
  for (ImmutableMap.Entry<String, SourceWithFlags> ent : sources.entrySet()) {
    String extension = Files.getFileExtension(ent.getKey());
    Optional<CxxSource.Type> type = CxxSource.Type.fromExtension(extension);
    if (!type.isPresent()) {
      throw new HumanReadableException("invalid extension \"%s\": %s", extension, ent.getKey());
    }
    cxxSources.put(
        ent.getKey(),
        CxxSource.of(type.get(), ent.getValue().getSourcePath(), ent.getValue().getFlags()));
  }

  return cxxSources.build();
}
 

@VisibleForTesting
ImmutableMap<String, MySQLDataSources> getEnqueueableShards() {
  ImmutableMap.Builder<String, MySQLDataSources> shardMapBuilder =
      new ImmutableMap.Builder<String, MySQLDataSources>();

  for (ImmutableMap.Entry<String, MySQLDataSources> shard : shardMapRef.get().entrySet()) {
    if (!shard.getValue().isDequeueOnly()) {
      shardMapBuilder.put(shard);
    }
  }

  return shardMapBuilder.build();
}
 

private static ImmutableMap.Entry<String, String> extractHostAndPort(String nextHostname, int defaultPort) {
    final String hostname;
    final String port;

    int idx = nextHostname.indexOf(':');
    if (idx > 0) {
        port = nextHostname.substring(idx + 1);
        hostname = nextHostname.substring(0, idx);
    } else {
        hostname = nextHostname;
        port = Integer.toString(defaultPort);
    }
    return Maps.immutableEntry(hostname, port);
}
 

/** Applies function designated in {@code expression} ([-|*]<float>) to value. */
private Integer applyOperator(@Nullable String expression, Supplier<Integer> firstOperandSupplier)
    throws OptionsParsingException {
  if (expression == null) {
    return firstOperandSupplier.get();
  }
  for (ImmutableMap.Entry<String, DoubleBinaryOperator> operator : OPERATORS.entrySet()) {
    if (expression.startsWith(operator.getKey())) {
      float secondOperand;
      try {
        secondOperand = Float.parseFloat(expression.substring(operator.getKey().length()));
      } catch (NumberFormatException e) {
        throw new OptionsParsingException(
            String.format("'%s is not a float", expression.substring(operator.getKey().length())),
            e);
      }
      return (int)
          Math.round(
              operator
                  .getValue()
                  .applyAsDouble((float) firstOperandSupplier.get(), secondOperand));
    }
  }
  // This should never happen because we've checked for a valid operator already.
  throw new OptionsParsingException(
      String.format("Parameter value '%s' does not contain a valid operator.", expression));
}
 

@Override
public void run() {
  final long runStartMillis = System.currentTimeMillis();
  for (ImmutableMap.Entry<String, MySQLDataSources> shard : getShardMap().entrySet()) {
    jobMonitorImpl(runStartMillis, shard, getMaxAutoRetries());
  }
}
 

private ImmutableMap.Entry<String, MySQLDataSources> getRandomShard() {
  ImmutableMap<String, MySQLDataSources> shardMap = shardMapRef.get();
  if (shardMap == null || shardMap.isEmpty()) {
    return null;
  }
  int rand = RANDOM.nextInt(shardMap.size());
  return shardMap.entrySet().asList().get(rand);
}
 

@Override
protected Set<String> getQueueNamesImpl() throws SQLException {
  ImmutableMap.Entry<String, MySQLDataSources> randomShard = getRandomShard();
  if (randomShard == null) {
    return Sets.newHashSet();
  }

  Connection conn = null;
  try {
    conn = randomShard.getValue().getGeneralDataSource().getConnection();
    return MySQLBackendUtils.getQueueNames(conn, randomShard.getKey());
  } finally {
    JdbcUtils.closeConnection(conn);
  }
}
 

private ImmutableList<SourcePath> generateMLBytecodeCompilation(
    ImmutableMap<Path, ImmutableList<Path>> mlSources) {
  ImmutableList.Builder<SourcePath> cmoFiles = ImmutableList.builder();

  Map<Path, ImmutableSortedSet<BuildRule>> sourceToRule = new HashMap<>();

  for (ImmutableMap.Entry<Path, ImmutableList<Path>> mlSource : mlSources.entrySet()) {
    generateSingleMLBytecodeCompilation(
        sourceToRule, cmoFiles, mlSource.getKey(), mlSources, ImmutableList.of());
  }
  return cmoFiles.build();
}
 

private ImmutableMap<ShardId, ShardSnapshotStatus> processWaitingShards(ImmutableMap<ShardId, ShardSnapshotStatus> snapshotShards, RoutingTable routingTable) {
    boolean snapshotChanged = false;
    ImmutableMap.Builder<ShardId, ShardSnapshotStatus> shards = ImmutableMap.builder();
    for (ImmutableMap.Entry<ShardId, ShardSnapshotStatus> shardEntry : snapshotShards.entrySet()) {
        ShardSnapshotStatus shardStatus = shardEntry.getValue();
        if (shardStatus.state() == State.WAITING) {
            ShardId shardId = shardEntry.getKey();
            IndexRoutingTable indexShardRoutingTable = routingTable.index(shardId.getIndex());
            if (indexShardRoutingTable != null) {
                IndexShardRoutingTable shardRouting = indexShardRoutingTable.shard(shardId.id());
                if (shardRouting != null && shardRouting.primaryShard() != null) {
                    if (shardRouting.primaryShard().started()) {
                        // Shard that we were waiting for has started on a node, let's process it
                        snapshotChanged = true;
                        logger.trace("starting shard that we were waiting for [{}] on node [{}]", shardEntry.getKey(), shardStatus.nodeId());
                        shards.put(shardEntry.getKey(), new ShardSnapshotStatus(shardRouting.primaryShard().currentNodeId()));
                        continue;
                    } else if (shardRouting.primaryShard().initializing() || shardRouting.primaryShard().relocating()) {
                        // Shard that we were waiting for hasn't started yet or still relocating - will continue to wait
                        shards.put(shardEntry);
                        continue;
                    }
                }
            }
            // Shard that we were waiting for went into unassigned state or disappeared - giving up
            snapshotChanged = true;
            logger.warn("failing snapshot of shard [{}] on unassigned shard [{}]", shardEntry.getKey(), shardStatus.nodeId());
            shards.put(shardEntry.getKey(), new ShardSnapshotStatus(shardStatus.nodeId(), State.FAILED, "shard is unassigned"));
        } else {
            shards.put(shardEntry);
        }
    }
    if (snapshotChanged) {
        return shards.build();
    } else {
        return null;
    }
}
 

private SortedMap<Path, Path> getReadOnlyBindMounts(
    BlazeDirectories blazeDirs, Path sandboxExecRoot) throws UserExecException {
  Path tmpPath = fileSystem.getPath("/tmp");
  final SortedMap<Path, Path> bindMounts = Maps.newTreeMap();
  if (blazeDirs.getWorkspace().startsWith(tmpPath)) {
    bindMounts.put(blazeDirs.getWorkspace(), blazeDirs.getWorkspace());
  }
  if (blazeDirs.getOutputBase().startsWith(tmpPath)) {
    bindMounts.put(blazeDirs.getOutputBase(), blazeDirs.getOutputBase());
  }
  for (ImmutableMap.Entry<String, String> additionalMountPath :
      getSandboxOptions().sandboxAdditionalMounts) {
    try {
      final Path mountTarget = fileSystem.getPath(additionalMountPath.getValue());
      // If source path is relative, treat it as a relative path inside the execution root
      final Path mountSource = sandboxExecRoot.getRelative(additionalMountPath.getKey());
      // If a target has more than one source path, the latter one will take effect.
      bindMounts.put(mountTarget, mountSource);
    } catch (IllegalArgumentException e) {
      throw new UserExecException(
          createFailureDetail(
              String.format("Error occurred when analyzing bind mount pairs. %s", e.getMessage()),
              Code.BIND_MOUNT_ANALYSIS_FAILURE));
    }
  }
  for (Path inaccessiblePath : getInaccessiblePaths()) {
    if (inaccessiblePath.isDirectory(Symlinks.NOFOLLOW)) {
      bindMounts.put(inaccessiblePath, inaccessibleHelperDir);
    } else {
      bindMounts.put(inaccessiblePath, inaccessibleHelperFile);
    }
  }
  validateBindMounts(bindMounts);
  return bindMounts;
}
 

public static ImmutableMap<String, String> withDefaults(
    ImmutableMap<String, String> map, ImmutableMap<String, String> defaults) {
  ImmutableMap.Builder<String, String> builder =
      ImmutableMap.<String, String>builder().putAll(map);
  for (ImmutableMap.Entry<String, String> entry : defaults.entrySet()) {
    if (!map.containsKey(entry.getKey())) {
      builder = builder.put(entry.getKey(), entry.getValue());
    }
  }
  return builder.build();
}
 

@Override
public ImmutableList<Step> getBuildSteps(
    BuildContext buildContext,
    ProjectFilesystem filesystem,
    OutputPathResolver outputPathResolver,
    BuildCellRelativePathFactory buildCellPathFactory) {

  ImmutableList.Builder<Step> steps = ImmutableList.builder();
  Path outputPath = outputPathResolver.resolvePath(output);
  Path srcPath = outputPathResolver.resolvePath(src);
  Path jarPath = outputPathResolver.resolvePath(outputJar);

  steps.add(MkdirStep.of(buildCellPathFactory.from(outputPath.getParent())));

  StringBuilder fileContents = new StringBuilder();
  for (ImmutableMap.Entry<String, String> entry : properties.entrySet()) {
    fileContents.append(entry.getKey()).append("=").append(entry.getValue()).append('\n');
  }

  steps.add(new WriteFileStep(filesystem, fileContents.toString(), outputPath, false));

  steps.add(RmStep.of(BuildCellRelativePath.of(jarPath)));
  steps.add(
      new ZipStep(
          filesystem, jarPath, ImmutableSet.of(), false, ZipCompressionLevel.NONE, srcPath));

  return steps.build();
}
 

static void forEachModuleParam(
    BuildTarget target,
    ActionGraphBuilder actionGraphBuilder,
    CxxPlatform cxxPlatform,
    String parameter,
    Path baseModule,
    Iterable<SourceSortedSet> inputs,
    BiConsumer<Path, SourcePath> consumer) {

  for (SourceSortedSet input : inputs) {
    ImmutableMap<String, SourcePath> namesAndSourcePaths;
    if (input.getUnnamedSources().isPresent()) {
      namesAndSourcePaths =
          actionGraphBuilder
              .getSourcePathResolver()
              .getSourcePathNames(target, parameter, input.getUnnamedSources().get());
    } else {
      namesAndSourcePaths = input.getNamedSources().get();
    }
    for (ImmutableMap.Entry<String, SourcePath> entry : namesAndSourcePaths.entrySet()) {
      consumer.accept(
          baseModule.resolve(entry.getKey()),
          CxxGenruleDescription.fixupSourcePath(
              actionGraphBuilder, cxxPlatform, entry.getValue()));
    }
  }
}
 

/**
 * Checks if any of the deleted indices are still recovering and fails recovery on the shards of these indices
 *
 * @param event cluster changed event
 */
private void processDeletedIndices(ClusterChangedEvent event) {
    RestoreInProgress restore = event.state().custom(RestoreInProgress.TYPE);
    if (restore == null) {
        // Not restoring - nothing to do
        return;
    }

    if (!event.indicesDeleted().isEmpty()) {
        // Some indices were deleted, let's make sure all indices that we are restoring still exist
        for (RestoreInProgress.Entry entry : restore.entries()) {
            List<ShardId> shardsToFail = null;
            for (ImmutableMap.Entry<ShardId, ShardRestoreStatus> shard : entry.shards().entrySet()) {
                if (!shard.getValue().state().completed()) {
                    if (!event.state().metaData().hasIndex(shard.getKey().getIndex())) {
                        if (shardsToFail == null) {
                            shardsToFail = new ArrayList<>();
                        }
                        shardsToFail.add(shard.getKey());
                    }
                }
            }
            if (shardsToFail != null) {
                for (ShardId shardId : shardsToFail) {
                    logger.trace("[{}] failing running shard restore [{}]", entry.snapshotId(), shardId);
                    updateRestoreStateOnMaster(new UpdateIndexShardRestoreStatusRequest(entry.snapshotId(), shardId, new ShardRestoreStatus(null, RestoreInProgress.State.FAILURE, "index was deleted")));
                }
            }
        }
    }
}
 

private static ImmutableMap<Path, ImmutableList<Path>> filterCurrentRuleInput(
    Set<Path> mlInput, ImmutableMap<Path, ImmutableList<Path>> deps) {
  ImmutableMap.Builder<Path, ImmutableList<Path>> builder = ImmutableMap.builder();
  for (ImmutableMap.Entry<Path, ImmutableList<Path>> entry : deps.entrySet()) {
    if (mlInput.contains(entry.getKey())) {
      builder.put(
          entry.getKey(),
          FluentIterable.from(entry.getValue()).filter(mlInput::contains).toList());
    }
  }
  return builder.build();
}
 

@Override
protected void createQueueImpl(final String queueName) throws Exception {
  for (ImmutableMap.Entry<String, MySQLDataSources> shard : shardMapRef.get().entrySet()) {
    Connection conn = null;
    String dbName = MySQLBackendUtils.constructDBName(queueName, shard.getKey());
    try {
      // We share the data source with enqueue.
      conn = shard.getValue().getGeneralDataSource().getConnection();

      // Create the database.
      JdbcUtils.executeUpdate(
          conn,
          String.format(MySQLQueries.CREATE_DATABASE, dbName));

      // Create the queue tables, one for each priority level.
      for (int priority = 1;
           priority <= numPriorityLevels;
           priority++) {
        JdbcUtils.executeUpdate(
            conn,
            String.format(
                MySQLQueries.CREATE_JOBS_TABLE,
                MySQLBackendUtils.constructJobsTableName(queueName, shard.getKey(), priority)));
      }
    } catch (SQLException e) {
      // If database already exists, then just ignore this and move onto the next shard.
      if (MySQLBackendUtils.isDatabaseAlreadyExistsException(e)) {
        continue;
      }

      // Wrap any other recognized exceptions as a PinLaterException.
      if (MySQLBackendUtils.isDatabaseNameTooLongException(e)) {
        throw new PinLaterException(ErrorCode.QUEUE_NAME_TOO_LONG,
            String.format(
                "Queue name is too long by %d characters. Attempted to create queue DB '%s'"
                    + " but it's longer than maximum allowed %d characters.",
                dbName.length() - MySQLBackendUtils.MYSQL_MAX_DB_NAME_LENGTH,
                dbName,
                MySQLBackendUtils.MYSQL_MAX_DB_NAME_LENGTH));
      }

      throw e;
    } finally {
      JdbcUtils.closeConnection(conn);
    }
  }
}
 

@Override
protected String enqueueSingleJob(final String queueName, final PinLaterJob job,
                                  int numAutoRetries) throws Exception {
  final double currentTimeSeconds = System.currentTimeMillis() / 1000.0;

  // Check whether the queue to enqueue exists.
  if ((queueNames.get() == null) || !queueNames.get().contains(queueName)) {
    reloadQueueNames();
    if (!queueNames.get().contains(queueName)) {
      Stats.incr("redis-queue-not-found-enqueue");
      throw new PinLaterException(ErrorCode.QUEUE_NOT_FOUND, "Queue not found: " + queueName);
    }
  }
  final ImmutableMap.Entry<String, RedisPools> shard = getRandomEnqueueableShard();
  if (shard == null) {
    throw new PinLaterException(ErrorCode.NO_HEALTHY_SHARDS, "Unable to find healthy shard");
  }
  try {
    return RedisUtils.executeWithConnection(
        shard.getValue().getGeneralRedisPool(),
        new Function<Jedis, String>() {
          @Override
          public String apply(Jedis conn) {
            String jobIdRedisKey = RedisBackendUtils.constructJobIdRedisKey(
                queueName, shard.getKey());
            String hashRedisKeyPrefix = RedisBackendUtils.constructHashRedisKeyPrefix(
                queueName, shard.getKey());
            String queueRedisKey = RedisBackendUtils.constructQueueRedisKey(
                queueName, shard.getKey(), job.getPriority(), PinLaterJobState.PENDING);
            List<String> keys = Lists.newArrayList(
                jobIdRedisKey, hashRedisKeyPrefix, queueRedisKey);
            double jobToRunTimestampSeconds;
            if (job.getRunAfterTimestampMillis() != 0) {
              jobToRunTimestampSeconds = job.getRunAfterTimestampMillis() / 1000.0;
            } else {
              jobToRunTimestampSeconds = currentTimeSeconds;
            }
            List<String> argv = Lists.newArrayList(
                BytesUtil.stringFromByteBuffer(ByteBuffer.wrap(job.getBody())),
                String.valueOf(job.getNumAttemptsAllowed()),
                String.valueOf(currentTimeSeconds),
                String.valueOf(jobToRunTimestampSeconds),
                RedisBackendUtils.truncateCustomStatus(job.getCustomStatus())
            );
            Long jobId = (Long) conn.eval(RedisLuaScripts.ENQUEUE_JOB, keys, argv);
            return new PinLaterJobDescriptor(
                queueName, shard.getKey(), job.getPriority(), jobId).toString();
          }
        });
  } catch (JedisConnectionException e) {
    if (numAutoRetries > 0) {
      // Retry the enqueue, potentially on a different shard.
      Stats.incr("enqueue-failures-retry");
      return enqueueSingleJob(queueName, job, numAutoRetries - 1);
    }
    String host = shard.getValue().getHost();
    Stats.incr("shard_connection_failed_" + host);
    LOG.error("Failed to get a redis connection.", e);
    throw new PinLaterException(ErrorCode.SHARD_CONNECTION_FAILED,
        String.format("Redis connection to %s failed", host));
  }
}
 

@Override
protected String enqueueSingleJob(String queueName, PinLaterJob job, int numAutoRetries)
    throws Exception {
  final long currentTimeMillis = System.currentTimeMillis();
  Connection conn = null;
  PreparedStatement stmt = null;
  ResultSet rs = null;
  final ImmutableMap.Entry<String, MySQLDataSources> shard = getRandomEnqueueableShard();
  try {
    conn = shard.getValue().getGeneralDataSource().getConnection();
    String jobsTableName =
        MySQLBackendUtils.constructJobsTableName(queueName, shard.getKey(), job.getPriority());
    stmt = conn.prepareStatement(
        String.format(MySQLQueries.ENQUEUE_INSERT, jobsTableName),
        Statement.RETURN_GENERATED_KEYS);
    stmt.setInt(1, PinLaterJobState.PENDING.getValue());
    stmt.setInt(2, job.getNumAttemptsAllowed());
    stmt.setInt(3, job.getNumAttemptsAllowed());
    stmt.setString(4, job.getCustomStatus());
    stmt.setTimestamp(5, new Timestamp(currentTimeMillis));
    stmt.setTimestamp(6, new Timestamp(job.isSetRunAfterTimestampMillis()
                                       ? job.getRunAfterTimestampMillis() : currentTimeMillis));
    stmt.setBytes(7, job.getBody());
    stmt.executeUpdate();
    rs = stmt.getGeneratedKeys();
    rs.next();
    return new PinLaterJobDescriptor(
        queueName, shard.getKey(), job.getPriority(), rs.getLong(1)).toString();
  } catch (SQLException e) {
    boolean shouldRetry = checkExceptionIsRetriable(e, shard.getKey(), "enqueue");
    if (shouldRetry && numAutoRetries > 0) {
      // Retry the enqueue, potentially on a different shard.
      Stats.incr("enqueue-failures-retry");
      return enqueueSingleJob(queueName, job, numAutoRetries - 1);
    }
    // Out of retries, throw the exception. Wrap it into a PinLaterException if the exception
    // is recognized and return the appropriate error code.
    if (MySQLBackendUtils.isDatabaseDoesNotExistException(e)) {
      throw new PinLaterException(ErrorCode.QUEUE_NOT_FOUND, "Queue not found: " + queueName);
    }
    throw e;
  } finally {
    JdbcUtils.closeResultSet(rs);
    JdbcUtils.closeStatement(stmt);
    JdbcUtils.closeConnection(conn);
  }
}
 

public List<String> build() {
  Preconditions.checkNotNull(sandboxExecRoot, "sandboxExecRoot must be set");
  Preconditions.checkState(!imageName.isEmpty(), "imageName must be set");
  Preconditions.checkState(!commandArguments.isEmpty(), "commandArguments must be set");

  ImmutableList.Builder<String> dockerCmdLine = ImmutableList.builder();

  dockerCmdLine.add(dockerClient.getPathString());
  dockerCmdLine.add("run");
  dockerCmdLine.add("--rm");
  if (createNetworkNamespace) {
    dockerCmdLine.add("--network=none");
  } else {
    dockerCmdLine.add("--network=host");
  }
  if (privileged) {
    dockerCmdLine.add("--privileged");
  }
  for (Map.Entry<String, String> env : environmentVariables.entrySet()) {
    dockerCmdLine.add("-e", env.getKey() + "=" + env.getValue());
  }
  PathFragment execRootInsideDocker =
      PathFragment.create("/execroot/").getRelative(sandboxExecRoot.getBaseName());
  dockerCmdLine.add(
      "-v", sandboxExecRoot.getPathString() + ":" + execRootInsideDocker.getPathString());
  dockerCmdLine.add("-w", execRootInsideDocker.getPathString());

  for (ImmutableMap.Entry<String, String> additionalMountPath : additionalMounts) {
    final String mountTarget = additionalMountPath.getValue();
    final String mountSource = additionalMountPath.getKey();
    dockerCmdLine.add("-v", mountSource + ":" + mountTarget);
  }

  StringBuilder uidGidFlagBuilder = new StringBuilder();
  if (uid != 0) {
    uidGidFlagBuilder.append(uid);
  }
  if (gid != 0) {
    uidGidFlagBuilder.append(":");
    uidGidFlagBuilder.append(gid);
  }
  String uidGidFlag = uidGidFlagBuilder.toString();
  if (!uidGidFlag.isEmpty()) {
    dockerCmdLine.add("-u", uidGidFlagBuilder.toString());
  }

  if (!commandId.isEmpty()) {
    dockerCmdLine.add("-l", "command_id=" + commandId);
  }
  if (uuid != null) {
    dockerCmdLine.add("--name", uuid.toString());
  }
  dockerCmdLine.add(imageName);
  dockerCmdLine.addAll(commandArguments);

  ProcessWrapper.CommandLineBuilder processWrapperCmdLine =
      processWrapper.commandLineBuilder(dockerCmdLine.build());
  if (timeout != null) {
    processWrapperCmdLine.setTimeout(timeout);
  }
  return processWrapperCmdLine.build();
}
 

private static void assertMountPair(
    ImmutableMap.Entry<String, String> pathPair, String source, String target) {
  assertThat(source).isEqualTo(pathPair.getKey());
  assertThat(target).isEqualTo(pathPair.getValue());
}