Java Code Examples for org.apache.reef.util.Optional#get()

/**
 * Launch application using the application DAG.
 * @param dag the application DAG.
 */
// When modifying the signature of this method, see CompilerTestUtil#compileDAG and make corresponding changes
public static void launchDAG(final DAG dag) {
  try {
    if (jobAndDriverConf == null || deployModeConf == null) {
      throw new RuntimeException("Configuration for launching driver is not ready");
    }
    final String serializedDAG = Base64.getEncoder().encodeToString(SerializationUtils.serialize(dag));
    final Configuration dagConf = TANG.newConfigurationBuilder()
        .bindNamedParameter(JobConf.SerializedDAG.class, serializedDAG)
        .build();
    // Launch and wait indefinitely for the job to finish
    final LauncherStatus launcherStatus =  DriverLauncher.getLauncher(deployModeConf)
        .run(Configurations.merge(jobAndDriverConf, dagConf));
    final Optional<Throwable> possibleError = launcherStatus.getError();
    if (possibleError.isPresent()) {
      throw new RuntimeException(possibleError.get());
    } else {
      LOG.info("Job successfully completed");
    }
  } catch (final InjectionException e) {
    throw new RuntimeException(e);
  }
}
 

private boolean cancel(final boolean mayInterruptIfRunning,
                       final Optional<Long> timeout,
                       final Optional<TimeUnit> unit) throws TimeoutException {
  if (isDone()) {
    return isCancelled();
  }

  vortexMaster.cancelTasklet(mayInterruptIfRunning, taskletId);

  try {
    if (timeout.isPresent() && unit.isPresent()) {
      if (!countDownLatch.await(timeout.get(), unit.get())) {
        throw new TimeoutException("Cancellation of the VortexFuture timed out. Timeout = " + timeout.get()
                + " in time units: " + unit.get());
      }
    } else {
      countDownLatch.await();
    }
  } catch (InterruptedException e) {
    e.printStackTrace();
    return false;
  }

  return isCancelled();
}
 

/**
 * Add a new tasklet to pendingTasklets.
 */
@Override
public <TInput, TOutput> VortexFuture<TOutput>
    enqueueTasklet(final VortexFunction<TInput, TOutput> function, final TInput input,
                   final Optional<FutureCallback<TOutput>> callback) {
  // TODO[REEF-500]: Simple duplicate Vortex Tasklet launch.
  final VortexFuture<TOutput> vortexFuture;
  final int id = taskletIdCounter.getAndIncrement();
  if (callback.isPresent()) {
    vortexFuture = new VortexFuture<>(executor, this, id, callback.get());
  } else {
    vortexFuture = new VortexFuture<>(executor, this, id);
  }

  final Tasklet tasklet = new Tasklet<>(id, Optional.<Integer>empty(), function, input, vortexFuture);
  putDelegate(Collections.singletonList(tasklet), vortexFuture);
  this.pendingTasklets.addLast(tasklet);

  return vortexFuture;
}
 

@Override
public V get(final K key, final Callable<V> valueFetcher) throws ExecutionException {
  // Before get, try to invalidate as many expired as possible
  expireEntries();

  final WrappedValue<V> newWrappedValue = new WrappedValue<>(valueFetcher, currentTime);
  final WrappedValue<V> existingWrappedValue = internalMap.putIfAbsent(key, newWrappedValue);

  if (existingWrappedValue == null) {
    // If absent, compute and return
    return newWrappedValue.loadAndGet();
  } else {
    final Optional<V> existingValue = existingWrappedValue.getValue();
    if (existingValue.isPresent()) {
      // If value already exists, get (without locking) and return
      return existingValue.get();
    } else {
      // If value is being computed, wait for computation to complete
      return existingWrappedValue.waitAndGet();
    }
  }
}
 

public static void shutdown() {
  // Trigger driver shutdown afterwards
  driverRPCServer.send(ControlMessage.ClientToDriverMessage.newBuilder()
    .setType(ControlMessage.ClientToDriverMessageType.DriverShutdown).build());
  // Wait for driver to naturally finish
  synchronized (driverLauncher) {
    while (!driverLauncher.getStatus().isDone()) {
      try {
        LOG.info("Wait for the driver to finish");
        driverLauncher.wait();
      } catch (final InterruptedException e) {
        LOG.warn(INTERRUPTED, e);
        // clean up state...
        Thread.currentThread().interrupt();
      }
    }
    LOG.info("Driver terminated");
  }

  // Close everything that's left
  driverRPCServer.shutdown();
  driverLauncher.close();
  isSetUp = false;
  final Optional<Throwable> possibleError = driverLauncher.getStatus().getError();
  if (possibleError.isPresent()) {
    throw new RuntimeException(possibleError.get());
  } else if (jobDoneLatch.getCount() > 0) {
    LOG.info("Job cancelled");
  } else {
    LOG.info("Job successfully completed");
  }
}
 

/**
 * Checks whether the FailedTask was caused by a ExpectedTaskException.
 *
 * @param failedTask
 * @throws org.apache.reef.tests.library.exceptions.DriverSideFailure if the FailedTask wasn't triggered by a
 *                                                                    ExpectedTaskException
 */

@Override
public void onNext(final FailedTask failedTask) {
  final Optional<Throwable> reasonOptional = failedTask.getReason();
  if (!reasonOptional.isPresent()) {
    throw new DriverSideFailure("Received a FailedTask, but it did not contain an exception.");
  } else if (!(Exceptions.getUltimateCause(reasonOptional.get()) instanceof ExpectedTaskException)) {
    throw new DriverSideFailure("Received a FailedTask, but the ExpectedTaskException isn't the ultimate cause.",
        reasonOptional.get());
  }
  failedTask.getActiveContext().get().close();
}
 

/**
 * This resource status message comes from the ResourceManager layer, telling me what it thinks
 * about the state of the resource executing an Evaluator. This method simply passes the message
 * off to the referenced EvaluatorManager
 *
 * @param resourceStatusEvent resource status message from the ResourceManager
 */
@Override
public void onNext(final ResourceStatusEvent resourceStatusEvent) {

  final String id = resourceStatusEvent.getIdentifier();
  final Optional<EvaluatorManager> evaluatorManager = this.evaluators.get(id);

  LOG.log(Level.FINEST, "Evaluator {0} status: {1}",
      new Object[] {evaluatorManager, resourceStatusEvent.getState()});

  if (evaluatorManager.isPresent()) {
    final EvaluatorManager evaluatorManagerImpl = evaluatorManager.get();
    evaluatorManagerImpl.onResourceStatusMessage(resourceStatusEvent);
    if (evaluatorManagerImpl.isClosed()) {
      this.evaluators.removeClosedEvaluator(evaluatorManagerImpl);
    }

  } else {

    if (this.evaluators.wasClosed(id)) {
      LOG.log(Level.WARNING,
          "Unexpected resource status from closed evaluator {0} with state {1}",
          new Object[] {id, resourceStatusEvent.getState()});
    }

    if (driverRestartManager.get().getEvaluatorRestartState(id).isFailedOrExpired()) {

      final EvaluatorManager previousEvaluatorManager = this.evaluatorManagerFactory
          .getNewEvaluatorManagerForEvaluatorFailedDuringDriverRestart(resourceStatusEvent);

      previousEvaluatorManager.onResourceStatusMessage(resourceStatusEvent);

    } else {
      throw new RuntimeException(
          "Unknown resource status from evaluator " + id + " with state " + resourceStatusEvent.getState());
    }
  }
}
 

/**
 * @return the path to "JAVA_HOME", if that is set. Optional.empty(), else.
 */
private static Optional<Path> getJavaHome() {
  final Optional<String> javaHome = getEnv("JAVA_HOME");

  if (javaHome.isPresent()) {
    final File javaHomeFile = new File(javaHome.get());
    if (javaHomeFile.exists()) {
      return Optional.of(javaHomeFile.toPath());
    }
  }
  return Optional.empty();
}
 

/**
/**
 * Checks the allocation queue for new allocations and if there are any
 * satisfies them.
 */
private void checkRequestQueue() {

  if (requestQueue.hasOutStandingRequests()) {
    final ResourceRequest resourceRequest = requestQueue.head();
    final ResourceRequestEvent requestEvent = resourceRequest.getRequestProto();
    final Optional<Container> cont = theContainers.allocateContainer(requestEvent);
    if (cont.isPresent()) {
      // Container has been allocated
      requestQueue.satisfyOne();
      final Container container = cont.get();
      // Tell the receivers about it
      final ResourceAllocationEvent alloc = ResourceEventImpl.newAllocationBuilder()
          .setIdentifier(container.getContainerID()).setNodeId(container.getNodeID())
          .setResourceMemory(container.getMemory()).setVirtualCores(container.getNumberOfCores())
          .setRackName(container.getRackName()).setRuntimeName(RuntimeIdentifier.RUNTIME_NAME).build();

      LOG.log(Level.FINEST, "Allocating container: {0}", container);
      this.allocationHandler.onNext(alloc);
      // update REEF
      this.sendRuntimeStatus();

      // Check whether we can satisfy another one.
      this.checkRequestQueue();
    } else {
      // could not allocate, update REEF
      this.sendRuntimeStatus();
    }
  } else {
    // done
    this.sendRuntimeStatus();
  }
}
 

/**
 * Add aggregate-able Tasklets to pendingTasklets.
 */
@Override
public <TInput, TOutput> VortexAggregateFuture<TInput, TOutput>
    enqueueTasklets(final VortexAggregateFunction<TOutput> aggregateFunction,
                    final VortexFunction<TInput, TOutput> vortexFunction,
                    final VortexAggregatePolicy policy,
                    final List<TInput> inputs,
                    final Optional<FutureCallback<AggregateResult<TInput, TOutput>>> callback) {
  final int aggregateFunctionId = aggregateIdCounter.getAndIncrement();
  aggregateFunctionRepository.put(aggregateFunctionId, aggregateFunction, policy);
  final List<Tasklet> tasklets = new ArrayList<>(inputs.size());
  final Map<Integer, TInput> taskletIdInputMap = new HashMap<>(inputs.size());

  for (final TInput input : inputs) {
    taskletIdInputMap.put(taskletIdCounter.getAndIncrement(), input);
  }

  final VortexAggregateFuture<TInput, TOutput> vortexAggregateFuture;
  if (callback.isPresent()) {
    vortexAggregateFuture = new VortexAggregateFuture<>(executor, taskletIdInputMap, callback.get());
  } else {
    vortexAggregateFuture = new VortexAggregateFuture<>(executor, taskletIdInputMap, null);
  }

  for (final Map.Entry<Integer, TInput> taskletIdInputEntry : taskletIdInputMap.entrySet()) {
    final Tasklet tasklet = new Tasklet<>(taskletIdInputEntry.getKey(), Optional.of(aggregateFunctionId),
        vortexFunction, taskletIdInputEntry.getValue(), vortexAggregateFuture);
    tasklets.add(tasklet);
    pendingTasklets.addLast(tasklet);
  }

  putDelegate(tasklets, vortexAggregateFuture);
  return vortexAggregateFuture;
}
 

/**
 * Remove the worker from runningWorkers and add back the lost tasklets to pendingTasklets.
 */
@Override
public void workerPreempted(final String id) {
  final Optional<Collection<Tasklet>> preemptedTasklets = runningWorkers.removeWorker(id);
  if (preemptedTasklets.isPresent()) {
    for (final Tasklet tasklet : preemptedTasklets.get()) {
      pendingTasklets.addFirst(tasklet);
    }
  }
}
 

@Override
public void registerDriverClient(
    final DriverClientRegistration request,
    final StreamObserver<Void> responseObserver) {
  LOG.log(Level.INFO, "driver client register");
  synchronized (GRPCDriverService.this) {
    try (ObserverCleanup cleanup = ObserverCleanup.of(responseObserver)) {
      if (request.hasException()) {
        LOG.log(Level.SEVERE, "Driver client initialization exception");
        final Optional<Throwable> optionalEx = parseException(request.getException());
        final Throwable ex;
        if (optionalEx.isPresent()) {
          ex = optionalEx.get();
        } else if (!request.getException().getData().isEmpty()) {
          ex = new NonSerializableException(request.getException().getMessage(),
              request.getException().getData().toByteArray());
        } else {
          ex = new RuntimeException(request.getException().getMessage());
        }
        stop(ex);
      } else {
        final ManagedChannel channel = ManagedChannelBuilder
            .forAddress(request.getHost(), request.getPort())
            .usePlaintext()
            .build();
        GRPCDriverService.this.clientStub = DriverClientGrpc.newFutureStub(channel);
        LOG.log(Level.INFO, "Driver has registered on port {0}", request.getPort());
      }
    } finally {
      GRPCDriverService.this.notifyAll();
    }
  }
}
 

void onResourceRequest(final ResourceRequestEvent resourceRequestEvent) {
  final Optional<String> node = selectNode(resourceRequestEvent);
  final String nodeId;

  if (node.isPresent()) {
    nodeId = node.get();
  } else {
    // Allocate new container
    nodeId = this.getNode() + ":" + String.valueOf(sshPortNum);
  }

  final String processID = nodeId + "-" + String.valueOf(System.currentTimeMillis());
  final File processFolder = new File(this.rootFolder, processID);

  final SshProcessContainer sshProcessContainer = new SshProcessContainer(errorHandlerRID, nodeId, processID,
      processFolder, resourceRequestEvent.getMemorySize().get(), resourceRequestEvent.getVirtualCores().get(),
      null, this.fileNames, this.nodeFolder, this.processObserver, this.containerThreads);

  this.containers.put(processID, sshProcessContainer);

  final ResourceAllocationEvent alloc = ResourceEventImpl.newAllocationBuilder()
      .setIdentifier(processID)
      .setNodeId(nodeId)
      .setResourceMemory(resourceRequestEvent.getMemorySize().get())
      .setVirtualCores(resourceRequestEvent.getVirtualCores().get())
      .setRuntimeName("STANDALONE")
      .build();
  reefEventHandlers.onResourceAllocation(alloc);

  // set the status as RUNNING.
  updateRuntimeStatus();
}
 

private void expireEntriesAtTime(final long now) {
  for (final Entry<K, WrappedValue<V>> entry : internalMap.entrySet()) {
    if (entry.getValue() != null) {
      final Optional<Long> writeTime = entry.getValue().getWriteTime();
      if (writeTime.isPresent() && writeTime.get() + timeoutMillis < now) {
        invalidate(entry.getKey());
      }
    }
  }
}